Merge remote-tracking branch 'origin/gc-backport'

.clippy.toml

@@ -1,2 +1,14 @@
 allow-print-in-tests = true
-allow-dbg-in-tests = true
+allow-dbg-in-tests = true
+
+# https://rust-lang.github.io/rust-clippy/master/index.html#disallowed_methods
+disallowed-methods = [
+    { path = "std::fs::File::create", reason = "use tycho_storage::FileDb instead" },
+    { path = "std::fs::File::create_new", reason = "use tycho_storage::FileDb instead" },
+    { path = "std::fs::File::open", reason = "use tycho_storage::FileDb instead" },
+    { path = "std::fs::File::options", reason = "use tycho_storage::FileDb instead" },
+]
+
+disallowed-types = [
+    { path = "std::fs::OpenOptions", reason = "use tycho_storage::FileDb instead" },
+]

Cargo.toml

@@ -114,6 +114,7 @@ char_lit_as_u8 = "warn"
 checked_conversions = "warn"
 dbg_macro = "warn"
 debug_assert_with_mut_call = "warn"
+disallowed_methods = "deny"
 doc_markdown = "warn"
 empty_enum = "warn"
 enum_glob_use = "warn"

README.md

@@ -4,11 +4,22 @@
 
 - [rust](https://rustup.rs/)
 - [just](https://pkgs.org/download/just)
+- [zstd](https://pkgs.org/download/zstd)
+- [clang](https://pkgs.org/download/clang)
+- [llvm](https://pkgs.org/download/llvm)
+- [fzf](https://github.com/junegunn/fzf?tab=readme-ov-file#using-linux-package-managers)
+  for just help menu
 
 # testing
 
 To run tests from ci:
 
+```bash
+just ci
+```
+
+To interactivly choose what to run:
+
 ```bash
 just
 ```

collator/src/manager/mod.rs

@@ -678,7 +678,7 @@ where
                 let mut test_subset = vec![];
                 for (i, keypair) in self.config.test_validators_keypairs.iter().enumerate() {
                     let val_descr = &subset[i];
-                    test_subset.push(ValidatorDescription {
+                    test_subset.push(everscale_types::models::ValidatorDescription {
                         public_key: keypair.public_key.to_bytes().into(),
                         adnl_addr: val_descr.adnl_addr,
                         weight: val_descr.weight,

justfile

@@ -1,27 +1,73 @@
-# Define default recipe
-default: fmt lint docs test
+default:
+   @just --choose
 
 install_fmt:
     rustup component add rustfmt --toolchain nightly
 
-# helpers
+
+integration_test_dir := justfile_directory() / ".scratch/integration_tests"
+integration_test_base_url := "https://tycho-test.broxus.cc"
+
+prepare_integration_tests:
+    #!/usr/bin/env bash
+    # Create the integration test directory if it does not exist
+    echo "Integration test directory: {{integration_test_dir}}"
+    mkdir -p {{integration_test_dir}}
+
+    # Always download the checksum file first to ensure it's the latest
+    echo "Downloading checksum file..."
+    curl --request GET -sL --url {{integration_test_base_url}}/states.tar.zst.sha256 --output {{integration_test_dir}}/states.tar.zst.sha256
+
+    # Check if the archive file exists
+    if [ -f {{integration_test_dir}}/states.tar.zst ]; then
+      # Verify the archive against the checksum
+      echo "Verifying existing archive against checksum..."
+      cd {{integration_test_dir}}
+      if sha256sum -c states.tar.zst.sha256; then
+          echo "Checksum matches. No need to download the archive."
+      else
+          echo "Checksum does not match. Downloading the archive..."
+          just _download_archive
+      fi
+      else
+          echo "Archive file does not exist. Downloading the archive..."
+          just _download_archive
+    fi
+
+_download_archive:
+    curl --request GET -L --url {{integration_test_base_url}}/states.tar.zst --output {{integration_test_dir}}/states.tar.zst
+
+clean_integration_tests:
+    rm -rf {{integration_test_dir}}
+
+
 fmt: install_fmt
     cargo +nightly fmt --all
 
 # ci checks
+ci: fmt lint docs test
+
 check_format: install_fmt
     cargo +nightly fmt --all -- --check
 
-lint: check_format
+lint:
     cargo clippy --all-targets --all-features --workspace
 
-docs: check_format
+docs:
     export RUSTDOCFLAGS=-D warnings
     cargo doc --no-deps --document-private-items --all-features --workspace
 
-test: lint
+test:
     cargo test --all-targets --all-features --workspace
 
+# runs all tests including ignored. Will take a lot of time to run
+integration_test: prepare_integration_tests
+    export RUST_BACKTRACE=1
+    export RUST_LIB_BACKTRACE=1
+    #cargo test -r --all-targets --all-features --workspace -- --ignored #uncomment this when all crates will compile ˙◠˙
+    # for now add tests one by one
+    RUST_LIB_BACKTRACE=1 RUST_BACKTRACE=1 cargo test -r --package tycho-storage --lib store::shard_state::replace_transaction::test::insert_and_delete_of_several_shards -- --ignored --exact --nocapture
+
 gen_network n: build_debug
     #!/usr/bin/env bash
     TEMP_DIR="./.temp"

simulator/Cargo.toml

@@ -22,5 +22,7 @@ rand = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 
+tycho-util = { workspace = true }
+
 [lints]
 workspace = true

simulator/src/compose.rs

@@ -173,7 +173,8 @@ impl ComposeRunner {
     pub fn get_running_nodes_list(&self) -> Result<Vec<String>> {
         let docker_compose_command = vec!["config".to_string(), "--services".to_string()];
         let output = self.execute_compose_command(&docker_compose_command)?;
-        let x = String::from_utf8(output.stdout)?.trim()
+        let x = String::from_utf8(output.stdout)?
+            .trim()
             .split("\n")
             .map(|x| x.to_string())
             .collect();

simulator/src/config.rs

@@ -1,6 +1,6 @@
-use anyhow::{Context, Result};
+use anyhow::Result;
 use std::path::PathBuf;
-use std::process::Command;
+use tycho_util::project_root;
 
 pub struct ServiceConfig {
     pub project_root: PathBuf,
@@ -11,17 +11,7 @@ pub struct ServiceConfig {
 
 impl ServiceConfig {
     pub fn new(network_subnet: String) -> Result<Self> {
-        let project_root = Command::new("git")
-            .arg("rev-parse")
-            .arg("--show-toplevel")
-            .output()?
-            .stdout;
-        // won't work on windows but we don't care
-        let project_root = PathBuf::from(
-            String::from_utf8(project_root)
-                .context("Invalid project root")?
-                .trim(),
-        );
+        let project_root = project_root()?;
         let scratch_dir = project_root.join(".scratch");
         Ok(Self {
             project_root,

storage/Cargo.toml

@@ -16,11 +16,12 @@ bytes = { workspace = true }
 bytesize = { workspace = true }
 crc = { workspace = true }
 dashmap = { workspace = true }
-everscale-types = { workspace = true }
+everscale-types = { workspace = true, features = ["tycho"] }
 fdlimit = { workspace = true }
 hex = { workspace = true }
 humantime = { workspace = true }
 libc = { workspace = true }
+metrics = { workspace = true }
 num-traits = { workspace = true }
 parking_lot = { workspace = true }
 parking_lot_core = { workspace = true }
@@ -42,14 +43,18 @@ tycho-block-util = { workspace = true }
 tycho-util = { workspace = true }
 
 [dev-dependencies]
+anyhow = { workspace = true, features = ["backtrace"] }
 base64 = { workspace = true }
 bytesize = { workspace = true }
 serde_json = { workspace = true }
+tempfile = { workspace = true }
+tokio = { workspace = true, features = ["full"] }
 tracing-appender = { workspace = true }
 tracing-subscriber = { workspace = true }
 tracing-test = { workspace = true }
-tempfile = { workspace = true }
-tokio = { version = "1", features = ["full"] }
+
+tycho-util = { workspace = true, features = ["test"] }
+tycho-storage = { workspace = true, features = ["test"] }
 
 [features]
 test = ["dep:tempfile"]

storage/src/db/file_db/mapped_file.rs

@@ -1,3 +1,4 @@
+#![allow(clippy::disallowed_types)]
 use std::fs::File;
 use std::os::fd::AsRawFd;
 use std::path::Path;

storage/src/db/file_db/mod.rs

@@ -1,8 +1,13 @@
+#![allow(clippy::disallowed_methods)]
+#![allow(clippy::disallowed_types)] // it's wrapper around Files so
+
 use std::fs::{File, OpenOptions};
 use std::os::fd::AsRawFd;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
+use anyhow::{Context, Result};
+
 pub use self::mapped_file::MappedFile;
 pub use self::temp_file::TempFile;
 
@@ -13,7 +18,21 @@ mod temp_file;
 pub struct FileDb(Arc<FileDbInner>);
 
 impl FileDb {
-    pub fn new<P>(root: P) -> Self
+    /// Creates a new `FileDb` instance.
+    /// If the `root` directory does not exist, it will be created.
+    pub fn new<P>(root: P) -> Result<Self>
+    where
+        P: AsRef<Path>,
+    {
+        std::fs::create_dir_all(root.as_ref())
+            .with_context(|| format!("failed to create {}", root.as_ref().display()))?;
+        Ok(Self(Arc::new(FileDbInner {
+            base_dir: root.as_ref().to_path_buf(),
+        })))
+    }
+
+    /// Creates a new `FileDb` without creating the root directory tree
+    pub fn new_readonly<P>(root: P) -> Self
     where
         P: AsRef<Path>,
     {
@@ -26,7 +45,7 @@ impl FileDb {
         &self.0.base_dir
     }
 
-    pub fn ensure_exists(&self) -> std::io::Result<()> {
+    pub fn create_if_not_exists(&self) -> std::io::Result<()> {
         std::fs::create_dir_all(&self.0.base_dir)
     }
 
@@ -46,12 +65,21 @@ impl FileDb {
         }
     }
 
-    pub fn subdir<P: AsRef<Path>>(&self, rel_path: P) -> Self {
+    /// Creates `FileDb` instance for a subdirectory of the current one.
+    /// **Note**: The subdirectory will not be created if it does not exist.
+    /// Use `create_subdir` to create it.
+    pub fn subdir_readonly<P: AsRef<Path>>(&self, rel_path: P) -> Self {
         Self(Arc::new(FileDbInner {
             base_dir: self.0.base_dir.join(rel_path),
         }))
     }
 
+    /// Creates `FileDb` instance for a subdirectory of the current one.
+    /// The subdirectory will be created if it does not exist.
+    pub fn create_subdir<P: AsRef<Path>>(&self, rel_path: P) -> Result<Self> {
+        Self::new(self.0.base_dir.join(rel_path))
+    }
+
     pub fn file_exists<P: AsRef<Path>>(&self, rel_path: P) -> bool {
         self.path().join(rel_path).is_file()
     }
@@ -72,24 +100,27 @@ pub struct FileBuilder {
 }
 
 impl FileBuilder {
-    pub fn open(&self) -> std::io::Result<File> {
-        let file = self.options.open(&self.path)?;
+    pub fn open(&self) -> Result<File> {
+        let file = self
+            .options
+            .open(&self.path)
+            .with_context(|| format!("failed to open {}", self.path.display()))?;
         if let Some(prealloc) = self.prealloc {
             alloc_file(&file, prealloc)?;
         }
         Ok(file)
     }
 
-    pub fn open_as_temp(&self) -> std::io::Result<TempFile> {
+    pub fn open_as_temp(&self) -> Result<TempFile> {
         let file = self.open()?;
         Ok(TempFile::new(self.path.clone(), file))
     }
 
-    pub fn open_as_mapped(&self) -> std::io::Result<MappedFile> {
-        match self.prealloc {
-            Some(length) => MappedFile::new(&self.path, length),
-            None => MappedFile::from_existing_file(self.open()?),
-        }
+    pub fn open_as_mapped(&self) -> Result<MappedFile> {
+        Ok(match self.prealloc {
+            Some(length) => MappedFile::new(&self.path, length)?,
+            None => MappedFile::from_existing_file(self.open()?)?,
+        })
     }
 
     pub fn append(&mut self, append: bool) -> &mut Self {
@@ -126,6 +157,10 @@ impl FileBuilder {
         self.prealloc = Some(prealloc);
         self
     }
+
+    pub fn path(&self) -> &Path {
+        &self.path
+    }
 }
 
 #[cfg(not(target_os = "macos"))]