fix(rand): remove timestamp randomness (#257)

- changed the imports across various files to match the current approach (sig.core.account; not @import)
- remove excess path naming of tests (test "bloom.bloom: ..." -> test "...")
- move `getWallclockMs` into `sig.time` (doesnt make sense to be in gossip)
- add seed parameter to `buildMessages` which is used for all random operations (prev seed was wallclock)
- add `seed` parameter to `PullRequestTask` (instead of using wallclock)
- add `now` parameter to `buildPullRequests` for more deterministic results (prev was reading wallclock)
- modifications to `test "test build pull requests"` to be a more deterministic test

note: the last two changes were aimed to make `test "test build pull requests"` more deterministic and fix #157 -- ive re-run the tests many times and it seems to be fixed

- remove unnecessary log lines in gossip
- add additional context to readme.md

metrics/README.md

@@ -4,13 +4,15 @@ original src: https://github.com/docker/awesome-compose/tree/master/prometheus-g
 
 requirements: 
 - `docker compose` 
+  - [https://docs.docker.com/engine/install/ubuntu/](https://docs.docker.com/engine/install/ubuntu/)
 - either mac or linux supported 
 
 modify `/etc/hosts` to include the following line: 
 ```
 127.0.0.1 prometheus
 ```
 
+
 ## Running 
 
 mac: `docker compose -f compose_mac.yaml up -d`

readme.md

@@ -33,6 +33,11 @@ metrics
 ├─ grafana/
 scripts/
 src/
+├─ main.zig # exec entrypoint 
+├─ sig.zig # library entrypoint
+├─ tests.zig 
+├─ fuzz.zig 
+├─ benchmarks.zig 
 ```
 
 

src/accountsdb/accounts_file.zig

@@ -2,14 +2,15 @@
 const std = @import("std");
 const sig = @import("../sig.zig");
 
-const Account = sig.core.Account;
-const writeIntLittleMem = sig.core.account.writeIntLittleMem;
-const Hash = sig.core.Hash;
-const Slot = sig.core.Slot;
-const Epoch = sig.core.Epoch;
-const Pubkey = sig.core.Pubkey;
+const Account = sig.core.account.Account;
+const Hash = sig.core.hash.Hash;
+const Slot = sig.core.time.Slot;
+const Epoch = sig.core.time.Epoch;
+const Pubkey = sig.core.pubkey.Pubkey;
 const AccountFileInfo = sig.accounts_db.snapshots.AccountFileInfo;
 
+const writeIntLittleMem = sig.core.account.writeIntLittleMem;
+
 /// Simple strictly-typed alias for an integer, used to represent a file ID.
 ///
 /// Analogous to [AccountsFileId](https://github.com/anza-xyz/agave/blob/4c921ca276bbd5997f809dec1dd3937fb06463cc/accounts-db/src/accounts_db.rs#L824)

src/accountsdb/bank.zig

@@ -1,5 +1,4 @@
 //! minimal logic for bank (still being built out)
-
 const std = @import("std");
 const sig = @import("../sig.zig");
 

src/accountsdb/db.zig

@@ -562,10 +562,9 @@ pub const AccountsDB = struct {
             // read accounts file
             var accounts_file = blk: {
                 const file_name_bounded = sig.utils.fmt.boundedFmt("{d}.{d}", .{ slot, file_info.id.toInt() });
-                errdefer std.debug.print("failed to open file: {s}\n", .{file_name_bounded.constSlice()});
 
                 const accounts_file_file = accounts_dir.openFile(file_name_bounded.constSlice(), .{ .mode = .read_write }) catch |err| {
-                    self.logger.errf("Failed to open accounts/{s}", .{file_name_bounded.constSlice()});
+                    self.logger.errf("Failed to open accounts/{s}: {s}", .{ file_name_bounded.constSlice(), @errorName(err) });
                     return err;
                 };
                 errdefer accounts_file_file.close();

src/accountsdb/fuzz_snapshot.zig

@@ -1,5 +1,4 @@
 const std = @import("std");
-
 const sig = @import("../sig.zig");
 
 const bincode = sig.bincode;

src/accountsdb/index.zig

@@ -1,18 +1,16 @@
 //! all index related structs (account ref, simd hashmap, …)
-
 const std = @import("std");
-const lib = @import("../sig.zig");
-
-const Slot = lib.core.time.Slot;
-const Pubkey = lib.core.pubkey.Pubkey;
-const FileId = lib.accounts_db.accounts_file.FileId;
-const RwMux = lib.sync.RwMux;
-
-const swiss_map = @import("swiss_map.zig");
-pub const SwissMapManaged = swiss_map.SwissMapManaged;
-pub const SwissMapUnmanaged = swiss_map.SwissMapUnmanaged;
-pub const BenchmarkSwissMap = swiss_map.BenchmarkSwissMap;
-pub const BenchHashMap = swiss_map.BenchHashMap;
+const sig = @import("../sig.zig");
+
+const Slot = sig.core.time.Slot;
+const Pubkey = sig.core.pubkey.Pubkey;
+const FileId = sig.accounts_db.accounts_file.FileId;
+const RwMux = sig.sync.RwMux;
+
+pub const SwissMapManaged = sig.accounts_db.swiss_map.SwissMapManaged;
+pub const SwissMapUnmanaged = sig.accounts_db.swiss_map.SwissMapUnmanaged;
+pub const BenchHashMap = sig.accounts_db.swiss_map.BenchHashMap;
+pub const BenchmarkSwissMap = sig.accounts_db.swiss_map.BenchmarkSwissMap;
 
 // for sync reasons we need a stable head with a lock
 pub const AccountReferenceHead = RwMux(struct {

src/accountsdb/lib.zig

@@ -8,6 +8,7 @@ pub const snapshots = @import("snapshots.zig");
 pub const sysvars = @import("sysvars.zig");
 pub const fuzz = @import("fuzz.zig");
 pub const fuzz_snapshot = @import("fuzz_snapshot.zig");
+pub const swiss_map = @import("swiss_map.zig");
 
 pub const AccountsDB = db.AccountsDB;
 pub const AllSnapshotFields = snapshots.AllSnapshotFields;

src/accountsdb/snapshots.zig

@@ -1854,7 +1854,7 @@ pub fn parallelUnpackZstdTarBall(
     );
 }
 
-test "test full snapshot path parsing" {
+test "full snapshot path parsing" {
     const full_snapshot_path = "snapshot-269-EAHHZCVccCdAoCXH8RWxvv9edcwjY2boqni9MJuh3TCn.tar.zst";
     const snapshot_info = try FullSnapshotFileInfo.fromString(full_snapshot_path);
 
@@ -1863,7 +1863,7 @@ test "test full snapshot path parsing" {
     try std.testing.expectEqual(.zstd, snapshot_info.compression);
 }
 
-test "test incremental snapshot path parsing" {
+test "incremental snapshot path parsing" {
     const path = "incremental-snapshot-269-307-4JLFzdaaqkSrmHs55bBDhZrQjHYZvqU1vCcQ5mP22pdB.tar.zst";
     const snapshot_info = try IncrementalSnapshotFileInfo.fromString(path);
 

src/bloom/bloom.zig

@@ -119,7 +119,7 @@ pub const Bloom = struct {
     }
 };
 
-test "bloom.bloom: helper fcns match rust" {
+test "helper methods match rust" {
     const n_bits = Bloom.numBits(100.2, 1e-5);
     try testing.expectEqual(@as(f64, 2402), n_bits);
 
@@ -131,7 +131,7 @@ test "bloom.bloom: helper fcns match rust" {
     defer bloom.deinit();
 }
 
-test "bloom.bloom: serializes/deserializes correctly" {
+test "serializes/deserializes correctly" {
     const bloom = Bloom.init(testing.allocator, 0, null);
 
     var buf: [10000]u8 = undefined;
@@ -145,7 +145,7 @@ test "bloom.bloom: serializes/deserializes correctly" {
     try testing.expect(bloom.num_bits_set == deserialized.num_bits_set);
 }
 
-test "bloom.bloom: serializes/deserializes correctly with set bits" {
+test "serializes/deserializes correctly with set bits" {
     var bloom = Bloom.init(testing.allocator, 128, null);
     try bloom.addKey(10);
     // required for memory leaks
@@ -160,7 +160,7 @@ test "bloom.bloom: serializes/deserializes correctly with set bits" {
     try testing.expect(bloom.num_bits_set == deserialized.num_bits_set);
 }
 
-test "bloom.bloom: rust: serialized bytes equal rust (one key)" {
+test "serialized bytes equal rust (one key)" {
     // note: need to init with len 2^i
     var bloom = Bloom.init(testing.allocator, 128, null);
     defer bloom.deinit();
@@ -177,7 +177,7 @@ test "bloom.bloom: rust: serialized bytes equal rust (one key)" {
     try testing.expectEqualSlices(u8, &rust_bytes, bytes[0..bytes.len]);
 }
 
-test "bloom.bloom: rust: serialized bytes equal rust (multiple keys)" {
+test "serialized bytes equal rust (multiple keys)" {
     var bloom = Bloom.init(testing.allocator, 128, null);
     defer bloom.deinit();
 

src/cmd/cmd.zig

@@ -28,7 +28,7 @@ const StatusCache = sig.accounts_db.StatusCache;
 const downloadSnapshotsFromGossip = sig.accounts_db.downloadSnapshotsFromGossip;
 const getOrInitIdentity = helpers.getOrInitIdentity;
 const globalRegistry = sig.prometheus.globalRegistry;
-const getWallclockMs = sig.gossip.getWallclockMs;
+const getWallclockMs = sig.time.getWallclockMs;
 const leaderScheduleFromBank = sig.core.leader_schedule.leaderScheduleFromBank;
 const parallelUnpackZstdTarBall = sig.accounts_db.parallelUnpackZstdTarBall;
 const parseLeaderSchedule = sig.core.leader_schedule.parseLeaderSchedule;
@@ -909,7 +909,7 @@ fn validateSnapshot() !void {
         allocator,
         app_base.logger,
         null,
-        false,
+        true,
         geyser_writer,
     );
     defer snapshot_result.deinit();

src/gossip/active_set.zig

@@ -12,7 +12,7 @@ const LegacyContactInfo = sig.gossip.data.LegacyContactInfo;
 const ThreadSafeContactInfo = sig.gossip.data.ThreadSafeContactInfo;
 const GossipTable = sig.gossip.table.GossipTable;
 
-const getWallclockMs = sig.gossip.getWallclockMs;
+const getWallclockMs = sig.time.getWallclockMs;
 const shuffleFirstN = sig.gossip.pull_request.shuffleFirstN;
 
 const NUM_ACTIVE_SET_ENTRIES: usize = 25;
@@ -128,7 +128,7 @@ pub const ActiveSet = struct {
     }
 };
 
-test "gossip.active_set: init/deinit" {
+test "init/denit" {
     const alloc = std.testing.allocator;
 
     const ThreadPool = @import("../sync/thread_pool.zig").ThreadPool;
@@ -154,8 +154,7 @@ test "gossip.active_set: init/deinit" {
 
     var active_set = ActiveSet.init(alloc);
     defer active_set.deinit();
-    var prng = std.Random.Xoshiro256.init(@intCast(std.time.milliTimestamp()));
-    try active_set.rotate(prng.random(), gossip_peers.items);
+    try active_set.rotate(rng.random(), gossip_peers.items);
 
     try std.testing.expect(active_set.len() == GOSSIP_PUSH_FANOUT);
 
@@ -175,7 +174,7 @@ test "gossip.active_set: init/deinit" {
     try std.testing.expectEqual(no_prune_fanout_len, fanout_with_prune.items.len + 1);
 }
 
-test "gossip.active_set: gracefully rotates with duplicate contact ids" {
+test "gracefully rotates with duplicate contact ids" {
     const alloc = std.testing.allocator;
 
     var rng = std.rand.DefaultPrng.init(100);
@@ -192,6 +191,6 @@ test "gossip.active_set: gracefully rotates with duplicate contact ids" {
 
     var active_set = ActiveSet.init(alloc);
     defer active_set.deinit();
-    var prng = std.Random.Xoshiro256.init(@intCast(std.time.milliTimestamp()));
+    var prng = std.rand.Xoshiro256.init(@intCast(std.time.milliTimestamp()));
     try active_set.rotate(prng.random(), gossip_peers.items);
 }