diff --git a/lib/dal/BUCK b/lib/dal/BUCK index 92cdd02636..be2eabb6e9 100644 --- a/lib/dal/BUCK +++ b/lib/dal/BUCK @@ -93,6 +93,7 @@ rust_test( "//lib/rebaser-server:rebaser-server", "//lib/si-events-rs:si-events", "//lib/si-frontend-types-rs:si-frontend-types", + "//lib/si-layer-cache:si-layer-cache", "//lib/si-pkg:si-pkg", "//lib/veritech-client:veritech-client", "//third-party/rust:chrono", diff --git a/lib/dal/examples/rebase/main.rs b/lib/dal/examples/rebase/main.rs index ccb5de2bb0..5721047bf5 100644 --- a/lib/dal/examples/rebase/main.rs +++ b/lib/dal/examples/rebase/main.rs @@ -1,17 +1,10 @@ use std::{env, fs::File, io::prelude::*}; use petgraph::{visit::EdgeRef, Direction::Incoming}; -use regex::bytes; use serde::de::DeserializeOwned; use si_layer_cache::db::serialize; -use dal::{ - workspace_snapshot::{ - graph::{correct_transforms::correct_transforms, RebaseBatch}, - node_weight::NodeWeight, - }, - WorkspaceSnapshotGraph, -}; +use dal::{workspace_snapshot::node_weight::NodeWeight, WorkspaceSnapshotGraph}; type Result = std::result::Result>; @@ -73,5 +66,3 @@ fn main() -> Result<()> { Ok(()) } - -// 01J2F7HKZFMTN6GVKXE73044AT diff --git a/lib/dal/src/layer_db_types/content_types.rs b/lib/dal/src/layer_db_types/content_types.rs index a72fdb90a1..5c086b5db6 100644 --- a/lib/dal/src/layer_db_types/content_types.rs +++ b/lib/dal/src/layer_db_types/content_types.rs @@ -37,6 +37,7 @@ pub type ContentTypeResult = Result; /// with postcard, DO *NOT* add new types to this list in alphabetical order. /// Add them to the *END* of the enum *ONLY*. #[derive(EnumDiscriminants, Serialize, Deserialize, Clone, PartialEq, Debug)] +#[strum_discriminants(derive(strum::EnumIter))] pub enum ContentTypes { Any(CasValue), AttributePrototype(AttributePrototypeContent), diff --git a/lib/dal/src/prop.rs b/lib/dal/src/prop.rs index 0dd224b081..fbfd5b4fec 100644 --- a/lib/dal/src/prop.rs +++ b/lib/dal/src/prop.rs @@ -255,7 +255,6 @@ impl From for PropPath { } } -#[remain::sorted] #[derive( AsRefStr, Clone, diff --git a/lib/dal/src/workspace_snapshot/edge_weight.rs b/lib/dal/src/workspace_snapshot/edge_weight.rs index a963bcce56..c4fc73d8b5 100644 --- a/lib/dal/src/workspace_snapshot/edge_weight.rs +++ b/lib/dal/src/workspace_snapshot/edge_weight.rs @@ -8,7 +8,7 @@ pub mod deprecated; /// This type is postcard serialized and new enum variants *MUST* be added to the end *ONLY*. #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash, EnumDiscriminants)] -#[strum_discriminants(derive(Hash, Serialize, Deserialize))] +#[strum_discriminants(derive(Hash, Serialize, Deserialize, strum::EnumIter))] pub enum EdgeWeightKind { Action, /// A function used by a [`SchemaVariant`] to perform an action that affects its resource diff --git a/lib/dal/src/workspace_snapshot/node_weight.rs b/lib/dal/src/workspace_snapshot/node_weight.rs index f53232c373..45f69ad26b 100644 --- a/lib/dal/src/workspace_snapshot/node_weight.rs +++ b/lib/dal/src/workspace_snapshot/node_weight.rs @@ -4,7 +4,7 @@ use finished_dependent_value_root_node_weight::FinishedDependentValueRootNodeWei use serde::{Deserialize, Serialize}; use si_events::{merkle_tree_hash::MerkleTreeHash, ulid::Ulid, ContentHash, EncryptedSecretKey}; use si_layer_cache::LayerDbError; -use strum::EnumDiscriminants; +use strum::{EnumDiscriminants, EnumIter}; use thiserror::Error; use traits::{CorrectExclusiveOutgoingEdge, CorrectTransforms, CorrectTransformsResult}; @@ -111,7 +111,7 @@ pub type NodeWeightResult = Result; /// **WARNING**: the order of this enum is important! Do not re-order elements. /// New variants must go at the end, even if it's not in lexical order! #[derive(Debug, Serialize, Deserialize, Clone, EnumDiscriminants, PartialEq, Eq)] -#[strum_discriminants(derive(strum::Display, Hash, Serialize, Deserialize))] +#[strum_discriminants(derive(strum::Display, Hash, Serialize, Deserialize, EnumIter))] pub enum NodeWeight { Action(ActionNodeWeight), ActionPrototype(ActionPrototypeNodeWeight), diff --git a/lib/dal/tests/integration_test/deserialize/mod.rs b/lib/dal/tests/integration_test/deserialize/mod.rs new file mode 100644 index 0000000000..f5e453beb9 --- /dev/null +++ b/lib/dal/tests/integration_test/deserialize/mod.rs @@ -0,0 +1,276 @@ +use std::io::Read; + +use dal::action::prototype::ActionKind; +use dal::func::FuncKind; +use dal::workspace_snapshot::content_address::ContentAddress; +use dal::workspace_snapshot::node_weight::category_node_weight::CategoryNodeKind; +use dal::workspace_snapshot::node_weight::traits::SiVersionedNodeWeight; +use dal::workspace_snapshot::node_weight::{ + ArgumentTargets, CategoryNodeWeight, NodeWeight, OrderingNodeWeight, +}; +use dal::{ + ContentHash, DalContext, EdgeWeight, EdgeWeightKind, EdgeWeightKindDiscriminants, + NodeWeightDiscriminants, SocketArity, WorkspaceSnapshotGraph, WorkspaceSnapshotGraphVCurrent, +}; +use dal::{PropKind, Ulid}; +use dal_test::test; +use si_events::EncryptedSecretKey; +use si_layer_cache::db::serialize; +use strum::IntoEnumIterator; + +const CURRENT_SERIALIZED_GRAPH_DIR_PATH: &str = "./lib/dal/tests"; +const CURRENT_SERIALIZED_GRAPH_FILENAME: &str = "serialization-test-data-2024-10-14.snapshot"; + +// If you're modifying this, you probably just added a new node or edge weight. Before you replace +// the snapshot with one that includes the new weights, ensure that your current code passes the +// deserialization test against the snapshot with the *old* weights. If it does, update the +// validation snapshot file. +#[allow(unused)] +fn make_me_one_with_everything(graph: &mut WorkspaceSnapshotGraphVCurrent) { + let mut node_indexes = vec![]; + + // For every enum that goes into a node weight, try to pick the last variant + // in the enum, so on the verification pass we will break if anything is + // added out of order + for node_kind in NodeWeightDiscriminants::iter() { + let weight = match node_kind { + NodeWeightDiscriminants::Action => { + NodeWeight::new_action(Ulid::new().into(), Ulid::new(), Ulid::new()) + } + NodeWeightDiscriminants::ActionPrototype => NodeWeight::new_action_prototype( + Ulid::new(), + Ulid::new(), + ActionKind::Update, + "foo".into(), + None, + ), + NodeWeightDiscriminants::AttributePrototypeArgument => { + NodeWeight::new_attribute_prototype_argument( + Ulid::new(), + Ulid::new(), + Some(ArgumentTargets { + source_component_id: Ulid::new().into(), + destination_component_id: Ulid::new().into(), + }), + ) + } + NodeWeightDiscriminants::AttributeValue => NodeWeight::new_attribute_value( + Ulid::new(), + Ulid::new(), + Some(ContentAddress::JsonValue(ContentHash::new( + "foo".as_bytes(), + ))), + Some(ContentAddress::JsonValue(ContentHash::new( + "foo".as_bytes(), + ))), + ), + NodeWeightDiscriminants::Category => NodeWeight::Category(CategoryNodeWeight::new( + Ulid::new(), + Ulid::new(), + CategoryNodeKind::DependentValueRoots, + )), + NodeWeightDiscriminants::Component => NodeWeight::new_component( + Ulid::new(), + Ulid::new(), + ContentHash::new("bar".as_bytes()), + ), + NodeWeightDiscriminants::Content => NodeWeight::new_content( + Ulid::new(), + Ulid::new(), + ContentAddress::ManagementPrototype(ContentHash::new("baz".as_bytes())), + ), + NodeWeightDiscriminants::DependentValueRoot => { + NodeWeight::new_dependent_value_root(Ulid::new(), Ulid::new(), Ulid::new()) + } + NodeWeightDiscriminants::Func => NodeWeight::new_func( + Ulid::new(), + Ulid::new(), + "foo", + FuncKind::Management, + ContentHash::new("quux".as_bytes()), + ), + NodeWeightDiscriminants::FuncArgument => NodeWeight::new_func_argument( + Ulid::new(), + Ulid::new(), + "arg", + ContentHash::new("the attitude era".as_bytes()), + ), + NodeWeightDiscriminants::Ordering => { + NodeWeight::Ordering(OrderingNodeWeight::new(Ulid::new(), Ulid::new())) + } + NodeWeightDiscriminants::Prop => NodeWeight::new_prop( + Ulid::new(), + Ulid::new(), + PropKind::String, + "foo", + ContentHash::new("bar".as_bytes()), + ), + NodeWeightDiscriminants::Secret => NodeWeight::new_secret( + Ulid::new(), + Ulid::new(), + EncryptedSecretKey::new("shhh".as_bytes()), + ContentHash::new("content".as_bytes()), + ), + NodeWeightDiscriminants::FinishedDependentValueRoot => { + NodeWeight::new_finished_dependent_value_root(Ulid::new(), Ulid::new(), Ulid::new()) + } + NodeWeightDiscriminants::InputSocket => NodeWeight::new_input_socket( + Ulid::new(), + Ulid::new(), + SocketArity::Many, + ContentHash::new("bar".as_bytes()), + ), + NodeWeightDiscriminants::SchemaVariant => NodeWeight::new_schema_variant( + Ulid::new(), + Ulid::new(), + false, + ContentHash::new("variant".as_bytes()), + ), + NodeWeightDiscriminants::ManagementPrototype => NodeWeight::new_management_prototype( + Ulid::new(), + Ulid::new(), + ContentHash::new("management".as_bytes()), + ), + }; + + let idx = graph.add_or_replace_node(weight).expect("add node"); + // Attach to root + graph + .add_edge( + graph.root(), + EdgeWeight::new(EdgeWeightKind::new_use()), + idx, + ) + .expect("add edge"); + node_indexes.push(idx); + } + + let mut last_node = 0; + + for edge_kind in EdgeWeightKindDiscriminants::iter() { + if last_node + 1 == node_indexes.len() { + last_node = 0; + }; + let edge_weight_kind = match edge_kind { + EdgeWeightKindDiscriminants::Action => EdgeWeightKind::Action, + EdgeWeightKindDiscriminants::ActionPrototype => EdgeWeightKind::ActionPrototype, + EdgeWeightKindDiscriminants::AuthenticationPrototype => { + EdgeWeightKind::AuthenticationPrototype + } + EdgeWeightKindDiscriminants::Contain => { + EdgeWeightKind::Contain(Some("foo".to_string())) + } + EdgeWeightKindDiscriminants::FrameContains => EdgeWeightKind::FrameContains, + EdgeWeightKindDiscriminants::Ordering => EdgeWeightKind::Ordering, + EdgeWeightKindDiscriminants::Ordinal => EdgeWeightKind::Ordinal, + EdgeWeightKindDiscriminants::Prop => EdgeWeightKind::Prop, + EdgeWeightKindDiscriminants::Prototype => { + EdgeWeightKind::Prototype(Some("bar".to_string())) + } + EdgeWeightKindDiscriminants::PrototypeArgument => EdgeWeightKind::PrototypeArgument, + EdgeWeightKindDiscriminants::PrototypeArgumentValue => { + EdgeWeightKind::PrototypeArgumentValue + } + EdgeWeightKindDiscriminants::Proxy => EdgeWeightKind::Proxy, + EdgeWeightKindDiscriminants::Root => EdgeWeightKind::Root, + EdgeWeightKindDiscriminants::Socket => EdgeWeightKind::Socket, + EdgeWeightKindDiscriminants::SocketValue => EdgeWeightKind::SocketValue, + EdgeWeightKindDiscriminants::Use => EdgeWeightKind::new_use(), + EdgeWeightKindDiscriminants::ValidationOutput => EdgeWeightKind::ValidationOutput, + EdgeWeightKindDiscriminants::ManagementPrototype => EdgeWeightKind::ManagementPrototype, + }; + + let edge_weight = EdgeWeight::new(edge_weight_kind); + + graph + .add_edge( + node_indexes[last_node], + edge_weight, + node_indexes[last_node + 1], + ) + .expect("add edge"); + + last_node += 1; + } +} + +// Run this test to produce a serialized version of the graph. Do this any time +// a breaking change in serialization occurs. +// +// cd to the root of "si" +// then run: +// +// $ buck2 run //lib/dal:test-integration -- write_deserialization_data --ignored +// +// Then delete the old copy of the graph, and replace the constant +// `CURRENT_SERIALIZED_GRAPH_FILENAME` with the filename of the new graph. +#[test] +#[ignore = "only run this when you want to produce a new serialized graph"] +async fn write_deserialization_data(_ctx: &DalContext) { + let mut graph = WorkspaceSnapshotGraphVCurrent::new().expect("make new"); + + make_me_one_with_everything(&mut graph); + + graph.cleanup_and_merkle_tree_hash().expect("hash it"); + + let real_graph = WorkspaceSnapshotGraph::V3(graph); + let serialized = serialize::to_vec(&real_graph).expect("serialize"); + + let date = chrono::Utc::now().format("%Y-%m-%d").to_string(); + + let mut file = std::fs::File::create(format!( + "{CURRENT_SERIALIZED_GRAPH_DIR_PATH}/serialization-test-data-{date}.snapshot" + )) + .expect("create file"); + file.write_all(&serialized).expect("write file"); +} + +#[test] +async fn graph_can_be_deserialized(_ctx: &DalContext) { + let mut file = std::fs::File::open(format!( + "{CURRENT_SERIALIZED_GRAPH_DIR_PATH}/{CURRENT_SERIALIZED_GRAPH_FILENAME}" + )) + .expect("open file"); + let mut bytes = vec![]; + file.read_to_end(&mut bytes).expect("able to read bytes"); + + let graph: WorkspaceSnapshotGraph = serialize::from_bytes(&bytes).expect("deserialize"); + + assert_eq!(18, graph.node_count()); + + // Where we can, verify that the enums on the node weights match what we expect + for (node_weight, _) in graph.nodes() { + match node_weight { + NodeWeight::Action(_) => {} + NodeWeight::ActionPrototype(action_prototype_node_weight) => { + assert_eq!(ActionKind::Update, action_prototype_node_weight.kind()); + } + NodeWeight::AttributePrototypeArgument(_) => {} + NodeWeight::AttributeValue(_) => {} + NodeWeight::Category(category_node_weight) => { + assert_eq!( + CategoryNodeKind::DependentValueRoots, + category_node_weight.kind() + ); + } + NodeWeight::Component(_) => {} + NodeWeight::Content(_) => {} + NodeWeight::DependentValueRoot(_) => {} + NodeWeight::Func(func_node_weight) => { + assert_eq!(FuncKind::Management, func_node_weight.func_kind()); + } + NodeWeight::FuncArgument(_) => {} + NodeWeight::Ordering(_) => {} + NodeWeight::Prop(prop_node_weight) => { + assert_eq!(PropKind::String, prop_node_weight.kind()); + } + NodeWeight::Secret(_) => {} + NodeWeight::FinishedDependentValueRoot(_) => {} + NodeWeight::InputSocket(input_socket_node_weight) => { + assert_eq!(SocketArity::Many, input_socket_node_weight.inner().arity()); + } + NodeWeight::SchemaVariant(_) => {} + NodeWeight::ManagementPrototype(_) => {} + } + } +} diff --git a/lib/dal/tests/integration_test/mod.rs b/lib/dal/tests/integration_test/mod.rs index abf436ae6a..3f13910c4d 100644 --- a/lib/dal/tests/integration_test/mod.rs +++ b/lib/dal/tests/integration_test/mod.rs @@ -7,6 +7,7 @@ mod component; mod connection; mod cycle_check_guard; mod dependent_values_update; +mod deserialize; mod diagram; mod frame; mod func; diff --git a/lib/dal/tests/serialization-test-data-2024-10-14.snapshot b/lib/dal/tests/serialization-test-data-2024-10-14.snapshot new file mode 100644 index 0000000000..520bdad4f8 Binary files /dev/null and b/lib/dal/tests/serialization-test-data-2024-10-14.snapshot differ