[6/n] [sled-agent] add remove_mupdate_override to OmicronSledConfig and friends (#8097)

This adds:

* data types
* db storage
* blueprint display
* blueprint diff display
* generation bump if there's a change to this field
* reconfigurator-cli tests for display and generation bump
* reconfigurator-cli support code including a couple of debug commands

In subsequent commits we'll make decisions based on this.

Author: sunshowers

Cargo.lock

@@ -21,6 +21,7 @@ humantime.workspace = true
 indent_write.workspace = true
 internal-dns-types.workspace = true
 itertools.workspace = true
+newtype-uuid.workspace = true
 nexus-inventory.workspace = true
 nexus-reconfigurator-blippy.workspace = true
 nexus-reconfigurator-planning.workspace = true

dev-tools/reconfigurator-cli/Cargo.toml

@@ -42,16 +42,16 @@ use omicron_common::api::external::Name;
 use omicron_common::policy::NEXUS_REDUNDANCY;
 use omicron_repl_utils::run_repl_from_file;
 use omicron_repl_utils::run_repl_on_stdin;
-use omicron_uuid_kinds::BlueprintUuid;
 use omicron_uuid_kinds::CollectionUuid;
 use omicron_uuid_kinds::GenericUuid;
 use omicron_uuid_kinds::OmicronZoneUuid;
 use omicron_uuid_kinds::ReconfiguratorSimUuid;
 use omicron_uuid_kinds::SledUuid;
 use omicron_uuid_kinds::VnicUuid;
+use omicron_uuid_kinds::{BlueprintUuid, MupdateOverrideUuid};
 use std::borrow::Cow;
 use std::collections::BTreeMap;
-use std::fmt::Write;
+use std::fmt::{self, Write};
 use std::io::IsTerminal;
 use std::str::FromStr;
 use swrite::{SWrite, swriteln};
@@ -204,7 +204,9 @@ fn process_command(
     let cmd_result = match command {
         Commands::SledList => cmd_sled_list(sim),
         Commands::SledAdd(args) => cmd_sled_add(sim, args),
+        Commands::SledRemove(args) => cmd_sled_remove(sim, args),
         Commands::SledShow(args) => cmd_sled_show(sim, args),
+        Commands::SledSetPolicy(args) => cmd_sled_set_policy(sim, args),
         Commands::SiloList => cmd_silo_list(sim),
         Commands::SiloAdd(args) => cmd_silo_add(sim, args),
         Commands::SiloRemove(args) => cmd_silo_remove(sim, args),
@@ -252,8 +254,12 @@ enum Commands {
     SledList,
     /// add a new sled
     SledAdd(SledAddArgs),
+    /// remove a sled from the system
+    SledRemove(SledRemoveArgs),
     /// show details about one sled
     SledShow(SledArgs),
+    /// set a sled's policy
+    SledSetPolicy(SledSetPolicyArgs),
 
     /// list silos
     SiloList,
@@ -308,6 +314,10 @@ enum Commands {
 struct SledAddArgs {
     /// id of the new sled
     sled_id: Option<SledUuid>,
+
+    /// number of disks or pools
+    #[clap(short = 'd', long, visible_alias = "npools", default_value_t = SledBuilder::DEFAULT_NPOOLS)]
+    ndisks: u8,
 }
 
 #[derive(Debug, Args)]
@@ -320,6 +330,53 @@ struct SledArgs {
     filter: SledFilter,
 }
 
+#[derive(Debug, Args)]
+struct SledSetPolicyArgs {
+    /// id of the sled
+    sled_id: SledUuid,
+
+    /// The policy to set for the sled
+    #[clap(value_enum)]
+    policy: SledPolicyOpt,
+}
+
+#[derive(Clone, Copy, Debug, ValueEnum)]
+enum SledPolicyOpt {
+    InService,
+    NonProvisionable,
+    Expunged,
+}
+
+impl fmt::Display for SledPolicyOpt {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            SledPolicyOpt::InService => write!(f, "in-service"),
+            SledPolicyOpt::NonProvisionable => write!(f, "non-provisionable"),
+            SledPolicyOpt::Expunged => write!(f, "expunged"),
+        }
+    }
+}
+
+impl From<SledPolicyOpt> for SledPolicy {
+    fn from(value: SledPolicyOpt) -> Self {
+        match value {
+            SledPolicyOpt::InService => SledPolicy::InService {
+                provision_policy: SledProvisionPolicy::Provisionable,
+            },
+            SledPolicyOpt::NonProvisionable => SledPolicy::InService {
+                provision_policy: SledProvisionPolicy::NonProvisionable,
+            },
+            SledPolicyOpt::Expunged => SledPolicy::Expunged,
+        }
+    }
+}
+
+#[derive(Debug, Args)]
+struct SledRemoveArgs {
+    /// id of the sled
+    sled_id: SledUuid,
+}
+
 #[derive(Debug, Args)]
 struct SiloAddRemoveArgs {
     /// name of the silo
@@ -374,6 +431,14 @@ enum BlueprintEditCommands {
         #[command(subcommand)]
         image_source: ImageSourceArgs,
     },
+    /// set the remove_mupdate_override field for a sled
+    SetRemoveMupdateOverride {
+        /// sled to set the field on
+        sled_id: SledUuid,
+
+        /// the UUID to set the field to, or "unset"
+        value: MupdateOverrideUuidOpt,
+    },
     /// expunge a zone
     ExpungeZone { zone_id: OmicronZoneUuid },
     /// configure an SP update
@@ -393,6 +458,34 @@ enum BlueprintEditCommands {
         /// baseboard serial number whose update to delete
         serial: String,
     },
+    /// debug commands that bypass normal checks
+    ///
+    /// These commands mutate the blueprint directly, bypassing higher-level
+    /// planner checks. They're meant for getting into weird states to test
+    /// against.
+    Debug {
+        #[command(subcommand)]
+        command: BlueprintEditDebugCommands,
+    },
+}
+
+#[derive(Debug, Subcommand)]
+enum BlueprintEditDebugCommands {
+    /// remove a sled from the blueprint
+    ///
+    /// This bypasses expungement and decommissioning checks, and simply drops
+    /// the sled from the blueprint.
+    RemoveSled {
+        /// the sled to remove
+        sled: SledUuid,
+    },
+
+    /// Bump a sled's generation number, even if nothing else about the sled has
+    /// changed.
+    ForceSledGenerationBump {
+        /// the sled to bump the sled-agent generation number of
+        sled: SledUuid,
+    },
 }
 
 #[derive(Clone, Debug)]
@@ -430,6 +523,42 @@ impl From<BlueprintIdOpt> for BlueprintId {
     }
 }
 
+/// Clap field for an optional mupdate override UUID.
+///
+/// This structure is similar to `Option`, but is specified separately to:
+///
+/// 1. Disable clap's magic around `Option`.
+/// 2. Provide a custom parser.
+///
+/// There are other ways to do both 1 and 2 (e.g. specify the type as
+/// `std::option::Option`), but when combined they're uglier than this.
+#[derive(Clone, Copy, Debug)]
+enum MupdateOverrideUuidOpt {
+    Unset,
+    Set(MupdateOverrideUuid),
+}
+
+impl FromStr for MupdateOverrideUuidOpt {
+    type Err = newtype_uuid::ParseError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        if s == "unset" || s == "none" {
+            Ok(MupdateOverrideUuidOpt::Unset)
+        } else {
+            Ok(MupdateOverrideUuidOpt::Set(s.parse::<MupdateOverrideUuid>()?))
+        }
+    }
+}
+
+impl From<MupdateOverrideUuidOpt> for Option<MupdateOverrideUuid> {
+    fn from(value: MupdateOverrideUuidOpt) -> Self {
+        match value {
+            MupdateOverrideUuidOpt::Unset => None,
+            MupdateOverrideUuidOpt::Set(uuid) => Some(uuid),
+        }
+    }
+}
+
 #[derive(Clone, Debug, Subcommand)]
 enum SpUpdateComponent {
     /// update the SP itself
@@ -673,7 +802,7 @@ fn cmd_sled_add(
 ) -> anyhow::Result<Option<String>> {
     let mut state = sim.current_state().to_mut();
     let sled_id = add.sled_id.unwrap_or_else(|| state.rng_mut().next_sled_id());
-    let new_sled = SledBuilder::new().id(sled_id);
+    let new_sled = SledBuilder::new().id(sled_id).npools(add.ndisks);
     state.system_mut().description_mut().sled(new_sled)?;
     sim.commit_and_bump(
         format!("reconfigurator-cli sled-add: {sled_id}"),
@@ -683,6 +812,24 @@ fn cmd_sled_add(
     Ok(Some(format!("added sled {}", sled_id)))
 }
 
+fn cmd_sled_remove(
+    sim: &mut ReconfiguratorSim,
+    args: SledRemoveArgs,
+) -> anyhow::Result<Option<String>> {
+    let mut state = sim.current_state().to_mut();
+    let sled_id = args.sled_id;
+    state
+        .system_mut()
+        .description_mut()
+        .sled_remove(sled_id)
+        .context("failed to remove sled")?;
+    sim.commit_and_bump(
+        format!("reconfigurator-cli sled-remove: {sled_id}"),
+        state,
+    );
+    Ok(Some(format!("removed sled {} from system", sled_id)))
+}
+
 fn cmd_sled_show(
     sim: &mut ReconfiguratorSim,
     args: SledArgs,
@@ -708,6 +855,25 @@ fn cmd_sled_show(
     Ok(Some(s))
 }
 
+fn cmd_sled_set_policy(
+    sim: &mut ReconfiguratorSim,
+    args: SledSetPolicyArgs,
+) -> anyhow::Result<Option<String>> {
+    let mut state = sim.current_state().to_mut();
+    state
+        .system_mut()
+        .description_mut()
+        .sled_set_policy(args.sled_id, args.policy.into())?;
+    sim.commit_and_bump(
+        format!(
+            "reconfigurator-cli sled-set-policy: {} to {}",
+            args.sled_id, args.policy,
+        ),
+        state,
+    );
+    Ok(Some(format!("set sled {} policy to {}", args.sled_id, args.policy)))
+}
+
 fn cmd_inventory_list(
     sim: &mut ReconfiguratorSim,
 ) -> anyhow::Result<Option<String>> {
@@ -925,6 +1091,19 @@ fn cmd_blueprint_edit(
                 .context("failed to add CockroachDB zone")?;
             format!("added CockroachDB zone to sled {}", sled_id)
         }
+        BlueprintEditCommands::SetRemoveMupdateOverride { sled_id, value } => {
+            builder
+                .sled_set_remove_mupdate_override(sled_id, value.into())
+                .context("failed to set remove_mupdate_override")?;
+            match value {
+                MupdateOverrideUuidOpt::Unset => {
+                    "unset remove_mupdate_override".to_owned()
+                }
+                MupdateOverrideUuidOpt::Set(uuid) => {
+                    format!("set remove_mupdate_override to {uuid}")
+                }
+            }
+        }
         BlueprintEditCommands::SetZoneImage { zone_id, image_source } => {
             let sled_id = sled_with_zone(&builder, &zone_id)?;
             let source = BlueprintZoneImageSource::from(image_source);
@@ -998,6 +1177,19 @@ fn cmd_blueprint_edit(
             builder.pending_mgs_update_delete(baseboard_id);
             format!("deleted configured update for serial {serial}")
         }
+        BlueprintEditCommands::Debug {
+            command: BlueprintEditDebugCommands::RemoveSled { sled },
+        } => {
+            builder.debug_sled_remove(sled)?;
+            format!("debug: removed sled {sled} from blueprint")
+        }
+        BlueprintEditCommands::Debug {
+            command:
+                BlueprintEditDebugCommands::ForceSledGenerationBump { sled },
+        } => {
+            builder.debug_sled_force_generation_bump(sled)?;
+            format!("debug: forced sled {sled} generation bump")
+        }
     };
 
     let mut new_blueprint = builder.build();

dev-tools/reconfigurator-cli/src/lib.rs

@@ -0,0 +1,41 @@
+# Load example system with 7 sleds:
+#
+# sled 0: unset -> unset (unchanged)
+# sled 1: unset -> set
+# sled 2: set -> unset
+# sled 3: set -> set (unchanged)
+# sled 4: set -> set (changed)
+# sled 5: set -> set (unchanged) but change something else
+# sled 6: set -> sled removed
+#
+# We'll also add another sled below (new_sled_id) with
+# remove_mupdate_override set.
+#
+# We don't need any zones for this test, so disable that to keep the
+# outputs minimal.
+
+load-example --nsleds 7 --ndisks-per-sled 0 --no-zones
+
+# Set the field on sleds 2-6 (0-indexed).
+blueprint-edit latest set-remove-mupdate-override 9a867dc9-d505-427f-9eff-cdb1d4d9bd73 00000000-0000-0000-0000-000000000000
+blueprint-edit latest set-remove-mupdate-override aff6c093-197d-42c5-ad80-9f10ba051a34 00000000-0000-0000-0000-000000000000
+blueprint-edit latest set-remove-mupdate-override b82ede02-399c-48c6-a1de-411df4fa49a7 00000000-0000-0000-0000-000000000000
+blueprint-edit latest set-remove-mupdate-override d81c6a84-79b8-4958-ae41-ea46c9b19763 00000000-0000-0000-0000-000000000000
+blueprint-edit latest set-remove-mupdate-override e96e226f-4ed9-4c01-91b9-69a9cd076c9e 00000000-0000-0000-0000-000000000000
+
+blueprint-show latest
+
+# Now make another blueprint, starting by adding a new sled and removing sled 6.
+sled-add --ndisks 0
+blueprint-edit latest debug remove-sled e96e226f-4ed9-4c01-91b9-69a9cd076c9e
+sled-remove e96e226f-4ed9-4c01-91b9-69a9cd076c9e
+inventory-generate
+
+# Edit sleds 1, 2, 4, 5, and the new one.
+blueprint-edit latest set-remove-mupdate-override 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 ffffffff-ffff-ffff-ffff-ffffffffffff
+blueprint-edit latest set-remove-mupdate-override 9a867dc9-d505-427f-9eff-cdb1d4d9bd73 unset
+blueprint-edit latest set-remove-mupdate-override b82ede02-399c-48c6-a1de-411df4fa49a7 ffffffff-ffff-ffff-ffff-ffffffffffff
+blueprint-edit latest debug force-sled-generation-bump d81c6a84-79b8-4958-ae41-ea46c9b19763
+blueprint-edit latest set-remove-mupdate-override 00320471-945d-413c-85e7-03e091a70b3c ffffffff-ffff-ffff-ffff-ffffffffffff
+
+blueprint-diff 7f976e0d-d2a5-4eeb-9e82-c82bc2824aba latest

dev-tools/reconfigurator-cli/tests/input/cmds-set-remove-mupdate-override.txt

@@ -2745,13 +2745,15 @@ edit contents of a blueprint directly
 Usage: blueprint-edit [OPTIONS] <BLUEPRINT_ID> <COMMAND>
 
 Commands:
-  add-nexus         add a Nexus instance to a particular sled
-  add-cockroach     add a CockroachDB instance to a particular sled
-  set-zone-image    set the image source for a zone
-  expunge-zone      expunge a zone
-  set-sp-update     configure an SP update
-  delete-sp-update  delete a configured SP update
-  help              Print this message or the help of the given subcommand(s)
+  add-nexus                    add a Nexus instance to a particular sled
+  add-cockroach                add a CockroachDB instance to a particular sled
+  set-zone-image               set the image source for a zone
+  set-remove-mupdate-override  set the remove_mupdate_override field for a sled
+  expunge-zone                 expunge a zone
+  set-sp-update                configure an SP update
+  delete-sp-update             delete a configured SP update
+  debug                        debug commands that bypass normal checks
+  help                         Print this message or the help of the given subcommand(s)
 
 Arguments:
   <BLUEPRINT_ID>  id of the blueprint to edit, "latest", or "target"