From 29389f8eb5a35e98da0ea69e5bdce87eb24aeea7 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 8 May 2025 15:44:22 +0200 Subject: [PATCH 01/27] use mainnet profile in mainnet artifacts build --- buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall | 3 +++ buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall | 3 +++ 2 files changed, 6 insertions(+) diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall index 97013973aca3..bc0b99315ea7 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall @@ -4,6 +4,8 @@ let Network = ../../Constants/Network.dhall let Artifacts = ../../Constants/Artifacts.dhall +let Profiles = ../../Constants/Profiles.dhall + let Pipeline = ../../Pipeline/Dsl.dhall let PipelineMode = ../../Pipeline/Mode.dhall @@ -27,5 +29,6 @@ in Pipeline.build , PipelineTag.Type.Stable ] , mode = PipelineMode.Type.Stable + , profile = Profiles.Type.Mainnet } ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall b/buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall index 8259ddc852ed..1b2dd6d90b82 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall @@ -6,6 +6,8 @@ let Network = ../../Constants/Network.dhall let Artifacts = ../../Constants/Artifacts.dhall +let Profiles = ../../Constants/Profiles.dhall + let Pipeline = ../../Pipeline/Dsl.dhall let PipelineMode = ../../Pipeline/Mode.dhall @@ -30,5 +32,6 @@ in Pipeline.build , PipelineTag.Type.Stable ] , mode = PipelineMode.Type.Stable + , profile = Profiles.Type.Mainnet } ) From d76c69f4bbc8051430a64d12102fd2f7b9bfe7db Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 8 May 2025 21:58:51 +0200 Subject: [PATCH 02/27] be more specific which profile is used --- buildkite/src/Command/Bench/Base.dhall | 2 +- buildkite/src/Command/Bench/LedgerApply.dhall | 2 +- buildkite/src/Command/DockerImage.dhall | 7 +++-- .../Command/HardforkPackageGeneration.dhall | 2 +- buildkite/src/Command/MinaArtifact.dhall | 5 ++-- .../src/Command/Promotion/PromoteDebian.dhall | 4 +-- .../src/Command/Promotion/PromoteDocker.dhall | 2 +- .../Command/Promotion/PromotePackages.dhall | 2 +- .../src/Command/Promotion/VerifyDebian.dhall | 2 +- .../Command/Promotion/VerifyPackages.dhall | 2 +- .../src/Command/Rosetta/Connectivity.dhall | 2 +- buildkite/src/Constants/Profiles.dhall | 26 ++++++++++++++----- .../src/Jobs/Promote/AutoPromoteNightly.dhall | 4 +-- ... MinaArtifactBullseyeBerkeleyDevnet.dhall} | 0 ...tBullseyeBerkeleyDevnetInstrumented.dhall} | 0 ...=> MinaArtifactBullseyeDevnetDevnet.dhall} | 0 ... MinaArtifactBullseyeMainnetMainnet.dhall} | 0 ... => MinaArtifactFocalBerkeleyDevnet.dhall} | 0 ...ll => MinaArtifactFocalDevnetDevnet.dhall} | 0 ... => MinaArtifactFocalMainnetMainnet.dhall} | 0 ...actOnlyDebianBullseyeBerkeleyDevnet.dhall} | 0 .../src/Jobs/Test/CheckGraphQLSchema.dhall | 2 +- buildkite/src/Jobs/Test/ConnectToDevnet.dhall | 2 +- .../src/Jobs/Test/ConnectToMainnet.dhall | 2 +- .../src/Jobs/Test/PatchArchiveTest.dhall | 2 +- buildkite/src/Jobs/Test/ReplayerTest.dhall | 2 +- .../Jobs/Test/RosettaIntegrationTests.dhall | 2 +- .../Test/RosettaIntegrationTestsLong.dhall | 2 +- buildkite/src/Jobs/Test/SingleNodeTest.dhall | 2 +- .../Jobs/Test/TestnetIntegrationTests.dhall | 4 +-- .../Test/TestnetIntegrationTestsLong.dhall | 4 +-- buildkite/src/Jobs/Test/VersionLint.dhall | 2 +- 32 files changed, 49 insertions(+), 39 deletions(-) rename buildkite/src/Jobs/Release/{MinaArtifactBullseyeBerkeley.dhall => MinaArtifactBullseyeBerkeleyDevnet.dhall} (100%) rename buildkite/src/Jobs/Release/{MinaArtifactBullseyeBerkeleyInstrumented.dhall => MinaArtifactBullseyeBerkeleyDevnetInstrumented.dhall} (100%) rename buildkite/src/Jobs/Release/{MinaArtifactBullseyeDevnet.dhall => MinaArtifactBullseyeDevnetDevnet.dhall} (100%) rename buildkite/src/Jobs/Release/{MinaArtifactBullseyeMainnet.dhall => MinaArtifactBullseyeMainnetMainnet.dhall} (100%) rename buildkite/src/Jobs/Release/{MinaArtifactFocalBerkeley.dhall => MinaArtifactFocalBerkeleyDevnet.dhall} (100%) rename buildkite/src/Jobs/Release/{MinaArtifactFocalDevnet.dhall => MinaArtifactFocalDevnetDevnet.dhall} (100%) rename buildkite/src/Jobs/Release/{MinaArtifactFocalMainnet.dhall => MinaArtifactFocalMainnetMainnet.dhall} (100%) rename buildkite/src/Jobs/Release/{MinaArtifactOnlyDebianBullseyeBerkeley.dhall => MinaArtifactOnlyDebianBullseyeBerkeleyDevnet.dhall} (100%) diff --git a/buildkite/src/Command/Bench/Base.dhall b/buildkite/src/Command/Bench/Base.dhall index c958a3b7b393..24a13303e4f9 100644 --- a/buildkite/src/Command/Bench/Base.dhall +++ b/buildkite/src/Command/Bench/Base.dhall @@ -52,7 +52,7 @@ let Spec = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye Network.Type.Berkeley - Profiles.Type.Standard + Profiles.Type.Devnet , additionalDirtyWhen = [] : List SelectFiles.Type , yellowThreshold = 0.1 , redThreshold = 0.2 diff --git a/buildkite/src/Command/Bench/LedgerApply.dhall b/buildkite/src/Command/Bench/LedgerApply.dhall index a38ef7c26f07..b6a01d0f189a 100644 --- a/buildkite/src/Command/Bench/LedgerApply.dhall +++ b/buildkite/src/Command/Bench/LedgerApply.dhall @@ -27,7 +27,7 @@ let dependsOn = (None Text) DebianVersions.DebVersion.Bullseye Network.Type.Berkeley - Profiles.Type.Standard + Profiles.Type.Devnet BuildFlags.Type.Instrumented "build" diff --git a/buildkite/src/Command/DockerImage.dhall b/buildkite/src/Command/DockerImage.dhall index dfd042260448..47bbda439bdd 100644 --- a/buildkite/src/Command/DockerImage.dhall +++ b/buildkite/src/Command/DockerImage.dhall @@ -56,7 +56,7 @@ let ReleaseSpec = , deb_codename = DebianVersions.DebVersion.Bullseye , deb_release = "\\\${MINA_DEB_RELEASE}" , deb_version = "\\\${MINA_DEB_VERSION}" - , deb_profile = Profiles.Type.Standard + , deb_profile = Profiles.Type.Devnet , build_flags = BuildFlags.Type.None , deb_repo = DebianRepo.Type.Local , docker_publish = DockerPublish.Type.Essential @@ -70,9 +70,8 @@ let ReleaseSpec = let stepKey = \(spec : ReleaseSpec.Type) -> "${Artifacts.lowerName - spec.service}${Profiles.toLabelSegment - spec.deb_profile}${BuildFlags.toLabelSegment - spec.build_flags}${spec.step_key_suffix}" + spec.service}${BuildFlags.toLabelSegment + spec.build_flags}${spec.step_key_suffix}" let stepLabel = \(spec : ReleaseSpec.Type) diff --git a/buildkite/src/Command/HardforkPackageGeneration.dhall b/buildkite/src/Command/HardforkPackageGeneration.dhall index 12d9e8623740..12cdb70d2386 100644 --- a/buildkite/src/Command/HardforkPackageGeneration.dhall +++ b/buildkite/src/Command/HardforkPackageGeneration.dhall @@ -53,7 +53,7 @@ let Spec = , genesis_timestamp = Some "2024-04-07T11:45:00Z" , config_json_gz_url = "https://storage.googleapis.com/o1labs-gitops-infrastructure/devnet/devnet-state-dump-3NK4eDgbkCjKj9fFUXVkrJXsfpfXzJySoAvrFJVCropPW7LLF14F-676026c4d4d2c18a76b357d6422a06f932c3ef4667a8fd88717f68b53fd6b2d7.json.gz" - , profile = Profiles.Type.Standard + , profile = Profiles.Type.Devnet , suffix = "hardfork" } } diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index d28ecbc4c18a..89aaa7d08aa7 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -54,7 +54,7 @@ let MinaBuildSpec = { prefix = "MinaArtifact" , artifacts = Artifacts.AllButTests , debVersion = DebianVersions.DebVersion.Bullseye - , profile = Profiles.Type.Standard + , profile = Profiles.Type.Devnet , buildFlags = BuildFlags.Type.None , network = Network.Type.Berkeley , toolchainSelectMode = Toolchain.SelectionMode.ByDebian @@ -86,8 +86,7 @@ let nameSuffix let keySuffix : MinaBuildSpec.Type -> Text = \(spec : MinaBuildSpec.Type) - -> "${Profiles.toLabelSegment spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}" + -> "${BuildFlags.toLabelSegment spec.buildFlags}" let build_artifacts : MinaBuildSpec.Type -> Command.Type diff --git a/buildkite/src/Command/Promotion/PromoteDebian.dhall b/buildkite/src/Command/Promotion/PromoteDebian.dhall index 66551ccfa6f7..9b4fb7dff747 100644 --- a/buildkite/src/Command/Promotion/PromoteDebian.dhall +++ b/buildkite/src/Command/Promotion/PromoteDebian.dhall @@ -50,7 +50,7 @@ let PromoteDebianSpec = , to_channel = DebianChannel.Type.Compatible , source_repo = DebianRepo.Type.Unstable , target_repo = DebianRepo.Type.Nightly - , profile = Profiles.Type.Standard + , profile = Profiles.Type.Devnet , remove_profile_from_name = False , step_key = "promote-debian-package" , if = None B/If @@ -68,7 +68,7 @@ let promoteDebianStep = then "--new-name ${Package.debianName spec.package - Profiles.Type.Standard + Profiles.Type.Devnet spec.network}" else "" diff --git a/buildkite/src/Command/Promotion/PromoteDocker.dhall b/buildkite/src/Command/Promotion/PromoteDocker.dhall index d2c19f88f4d4..4121f2e8f330 100644 --- a/buildkite/src/Command/Promotion/PromoteDocker.dhall +++ b/buildkite/src/Command/Promotion/PromoteDocker.dhall @@ -40,7 +40,7 @@ let PromoteDockerSpec = , version = "" , new_tags = [] : List Text , step_key = "promote-docker" - , profile = Profiles.Type.Standard + , profile = Profiles.Type.Devnet , network = Network.Type.Devnet , codename = DebianVersions.DebVersion.Bullseye , if = None B/If diff --git a/buildkite/src/Command/Promotion/PromotePackages.dhall b/buildkite/src/Command/Promotion/PromotePackages.dhall index 128d2b098403..d30878317315 100644 --- a/buildkite/src/Command/Promotion/PromotePackages.dhall +++ b/buildkite/src/Command/Promotion/PromotePackages.dhall @@ -70,7 +70,7 @@ let PromotePackagesSpec = , source_debian_repo = DebianRepo.Type.Unstable , target_debian_repo = DebianRepo.Type.Nightly , architecture = "amd64" - , profile = Profiles.Type.Standard + , profile = Profiles.Type.Devnet , network = Network.Type.Mainnet , codenames = [] : List DebianVersions.DebVersion , from_channel = DebianChannel.Type.Unstable diff --git a/buildkite/src/Command/Promotion/VerifyDebian.dhall b/buildkite/src/Command/Promotion/VerifyDebian.dhall index e178b8fbc694..df93895e0f6b 100644 --- a/buildkite/src/Command/Promotion/VerifyDebian.dhall +++ b/buildkite/src/Command/Promotion/VerifyDebian.dhall @@ -23,7 +23,7 @@ let promoteDebianVerificationStep = then "${Package.debianName spec.package - Profiles.Type.Standard + Profiles.Type.Devnet spec.network}" else Package.debianName spec.package spec.profile spec.network diff --git a/buildkite/src/Command/Promotion/VerifyPackages.dhall b/buildkite/src/Command/Promotion/VerifyPackages.dhall index c1010413f23c..f38797110c35 100644 --- a/buildkite/src/Command/Promotion/VerifyPackages.dhall +++ b/buildkite/src/Command/Promotion/VerifyPackages.dhall @@ -66,7 +66,7 @@ let VerifyPackagesSpec = , dockers = [] : List Artifact.Type , new_debian_version = "\\\\\$MINA_DEB_VERSION" , debian_repo = DebianRepo.Type.Local - , profile = Profiles.Type.Standard + , profile = Profiles.Type.Devnet , network = Network.Type.Mainnet , codenames = [] : List DebianVersions.DebVersion , channel = DebianChannel.Type.Compatible diff --git a/buildkite/src/Command/Rosetta/Connectivity.dhall b/buildkite/src/Command/Rosetta/Connectivity.dhall index 0335aeeced16..00782b5f36c1 100644 --- a/buildkite/src/Command/Rosetta/Connectivity.dhall +++ b/buildkite/src/Command/Rosetta/Connectivity.dhall @@ -70,7 +70,7 @@ let command Dockers.dependsOn spec.dockerType spec.network - Profiles.Type.Standard + (Profiles.fromNetwork spec.network) Artifacts.Type.Rosetta } diff --git a/buildkite/src/Constants/Profiles.dhall b/buildkite/src/Constants/Profiles.dhall index cf7cb3da71ac..54b42e5db308 100644 --- a/buildkite/src/Constants/Profiles.dhall +++ b/buildkite/src/Constants/Profiles.dhall @@ -1,11 +1,13 @@ +let Network = ./Network.dhall + let Profile : Type - = < Standard | Mainnet | Lightnet | Hardfork | Dev > + = < Devnet | Mainnet | Lightnet | Hardfork | Dev > let capitalName = \(profile : Profile) -> merge - { Standard = "Standard" + { Devnet = "Devnet" , Mainnet = "Mainnet" , Lightnet = "Lightnet" , Hardfork = "Hardfork" @@ -16,7 +18,7 @@ let capitalName = let lowerName = \(profile : Profile) -> merge - { Standard = "standard" + { Devnet = "devnet" , Mainnet = "mainnet" , Lightnet = "lightnet" , Hardfork = "hardfork" @@ -27,7 +29,7 @@ let lowerName = let duneProfile = \(profile : Profile) -> merge - { Standard = "devnet" + { Devnet = "devnet" , Mainnet = "mainnet" , Lightnet = "lightnet" , Hardfork = "hardfork" @@ -35,10 +37,19 @@ let duneProfile = } profile +let fromNetwork = + \(network : Network.Type) + -> merge + { Devnet = Profile.Devnet + , Mainnet = Profile.Mainnet + , Berkeley = Profile.Devnet + } + network + let toSuffixUppercase = \(profile : Profile) -> merge - { Standard = "" + { Devnet = "Devnet" , Mainnet = "Mainnet" , Lightnet = "Lightnet" , Hardfork = "Hardfork" @@ -49,7 +60,7 @@ let toSuffixUppercase = let toSuffixLowercase = \(profile : Profile) -> merge - { Standard = "" + { Devnet = "devnet" , Mainnet = "mainnet" , Lightnet = "lightnet" , Hardfork = "hardfork" @@ -60,7 +71,7 @@ let toSuffixLowercase = let toLabelSegment = \(profile : Profile) -> merge - { Standard = "" + { Devnet = "devnet" , Mainnet = "-mainnet" , Lightnet = "-lightnet" , Hardfork = "-hardfork" @@ -73,6 +84,7 @@ in { Type = Profile , lowerName = lowerName , duneProfile = duneProfile , toSuffixUppercase = toSuffixUppercase + , fromNetwork = fromNetwork , toSuffixLowercase = toSuffixLowercase , toLabelSegment = toLabelSegment } diff --git a/buildkite/src/Jobs/Promote/AutoPromoteNightly.dhall b/buildkite/src/Jobs/Promote/AutoPromoteNightly.dhall index 1ae82d7f0a9a..25ce2bc5798f 100644 --- a/buildkite/src/Jobs/Promote/AutoPromoteNightly.dhall +++ b/buildkite/src/Jobs/Promote/AutoPromoteNightly.dhall @@ -53,7 +53,7 @@ let promotePackages = , version = "\\\${FROM_VERSION_MANUAL:-\\\${MINA_DEB_VERSION}}" , architecture = "amd64" , new_debian_version = "\\\$(date \"+%Y%m%d\")" - , profile = Profiles.Type.Standard + , profile = Profiles.Type.Devnet , network = Network.Type.Devnet , codenames = [ DebianVersions.DebVersion.Bullseye, DebianVersions.DebVersion.Focal ] @@ -70,7 +70,7 @@ let verifyPackages = , debians = [] : List DebianPackage.Type , dockers = [ Artifacts.Type.Daemon, Artifacts.Type.Archive ] , new_debian_version = "\\\$(date \"+%Y%m%d\")" - , profile = Profiles.Type.Standard + , profile = Profiles.Type.Devnet , network = Network.Type.Devnet , codenames = [ DebianVersions.DebVersion.Bullseye, DebianVersions.DebVersion.Focal ] diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeley.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeleyDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeley.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeleyDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeleyInstrumented.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeleyDevnetInstrumented.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeleyInstrumented.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeleyDevnetInstrumented.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnet.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnet.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnetMainnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnetMainnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactFocalBerkeley.dhall b/buildkite/src/Jobs/Release/MinaArtifactFocalBerkeleyDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactFocalBerkeley.dhall rename to buildkite/src/Jobs/Release/MinaArtifactFocalBerkeleyDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactFocalDevnet.dhall b/buildkite/src/Jobs/Release/MinaArtifactFocalDevnetDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactFocalDevnet.dhall rename to buildkite/src/Jobs/Release/MinaArtifactFocalDevnetDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall b/buildkite/src/Jobs/Release/MinaArtifactFocalMainnetMainnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall rename to buildkite/src/Jobs/Release/MinaArtifactFocalMainnetMainnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseyeBerkeley.dhall b/buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseyeBerkeleyDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseyeBerkeley.dhall rename to buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseyeBerkeleyDevnet.dhall diff --git a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall index bffabd5a6537..5b27a6fe530e 100644 --- a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall +++ b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall @@ -18,7 +18,7 @@ let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye Network.Type.Berkeley - Profiles.Type.Standard + Profiles.Type.Devnet in Pipeline.build Pipeline.Config::{ diff --git a/buildkite/src/Jobs/Test/ConnectToDevnet.dhall b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall index c7e913811f48..45846d73a28b 100644 --- a/buildkite/src/Jobs/Test/ConnectToDevnet.dhall +++ b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall @@ -28,7 +28,7 @@ let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye network - Profiles.Type.Standard + Profiles.Type.Devnet Artifacts.Type.Daemon in Pipeline.build diff --git a/buildkite/src/Jobs/Test/ConnectToMainnet.dhall b/buildkite/src/Jobs/Test/ConnectToMainnet.dhall index a07829e9d8b7..7359ef59ff43 100644 --- a/buildkite/src/Jobs/Test/ConnectToMainnet.dhall +++ b/buildkite/src/Jobs/Test/ConnectToMainnet.dhall @@ -28,7 +28,7 @@ let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye network - Profiles.Type.Standard + Profiles.Type.Mainnet Artifacts.Type.Daemon in Pipeline.build diff --git a/buildkite/src/Jobs/Test/PatchArchiveTest.dhall b/buildkite/src/Jobs/Test/PatchArchiveTest.dhall index 6131cc2feb0e..8b3048ad6da7 100644 --- a/buildkite/src/Jobs/Test/PatchArchiveTest.dhall +++ b/buildkite/src/Jobs/Test/PatchArchiveTest.dhall @@ -20,7 +20,7 @@ let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye Network.Type.Berkeley - Profiles.Type.Standard + Profiles.Type.Devnet Artifacts.Type.FunctionalTestSuite in Pipeline.build diff --git a/buildkite/src/Jobs/Test/ReplayerTest.dhall b/buildkite/src/Jobs/Test/ReplayerTest.dhall index 3986b8f57d99..717d6c4f3315 100644 --- a/buildkite/src/Jobs/Test/ReplayerTest.dhall +++ b/buildkite/src/Jobs/Test/ReplayerTest.dhall @@ -20,7 +20,7 @@ let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye Network.Type.Berkeley - Profiles.Type.Standard + Profiles.Type.Devnet Artifacts.Type.FunctionalTestSuite in Pipeline.build diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index 70868cd62453..30e0a6a3f6f8 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -75,7 +75,7 @@ in Pipeline.build Dockers.dependsOn Dockers.Type.Bullseye network - Profiles.Type.Standard + Profiles.Type.Devnet Artifacts.Type.Rosetta } ] diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall index a67a439b17cb..a6e9179c01e8 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall @@ -71,7 +71,7 @@ in Pipeline.build Dockers.dependsOn Dockers.Type.Bullseye network - Profiles.Type.Standard + Profiles.Type.Devnet Artifacts.Type.Rosetta } ] diff --git a/buildkite/src/Jobs/Test/SingleNodeTest.dhall b/buildkite/src/Jobs/Test/SingleNodeTest.dhall index 9a695bf41981..348aef0be5d7 100644 --- a/buildkite/src/Jobs/Test/SingleNodeTest.dhall +++ b/buildkite/src/Jobs/Test/SingleNodeTest.dhall @@ -28,7 +28,7 @@ let dependsOn = # DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye Network.Type.Berkeley - Profiles.Type.Standard + Profiles.Type.Devnet let buildTestCmd : Size -> Command.Type diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall index b03adbe28d7b..26fb8b600029 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall @@ -22,12 +22,12 @@ let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye Network.Type.Berkeley - Profiles.Type.Standard + Profiles.Type.Devnet Artifacts.Type.Daemon # Dockers.dependsOn Dockers.Type.Bullseye Network.Type.Berkeley - Profiles.Type.Standard + Profiles.Type.Devnet Artifacts.Type.Archive in Pipeline.build diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall index 420d92021a85..698d4df504af 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall @@ -22,12 +22,12 @@ let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye Network.Type.Berkeley - Profiles.Type.Standard + Profiles.Type.Devnet Artifacts.Type.Daemon # Dockers.dependsOn Dockers.Type.Bullseye Network.Type.Berkeley - Profiles.Type.Standard + Profiles.Type.Devnet Artifacts.Type.Archive in Pipeline.build diff --git a/buildkite/src/Jobs/Test/VersionLint.dhall b/buildkite/src/Jobs/Test/VersionLint.dhall index 9ebc8a9f6253..0f795abe30b6 100644 --- a/buildkite/src/Jobs/Test/VersionLint.dhall +++ b/buildkite/src/Jobs/Test/VersionLint.dhall @@ -28,7 +28,7 @@ let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye Network.Type.Berkeley - Profiles.Type.Standard + Profiles.Type.Devnet let buildTestCmd : Text -> Size -> List Command.TaggedKey.Type -> B/SoftFail -> Command.Type From 7e34dfa6aede3b9e2073b7a7466c6d387416f7d5 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 8 May 2025 22:43:05 +0200 Subject: [PATCH 03/27] build archive for 3 networks --- dockerfiles/Dockerfile-mina-archive | 3 +- scripts/debian/build.sh | 4 ++- scripts/debian/builder-helpers.sh | 56 +++++++++++++++++++++++------ 3 files changed, 51 insertions(+), 12 deletions(-) diff --git a/dockerfiles/Dockerfile-mina-archive b/dockerfiles/Dockerfile-mina-archive index f340607992f6..20e9d7aa9b5f 100644 --- a/dockerfiles/Dockerfile-mina-archive +++ b/dockerfiles/Dockerfile-mina-archive @@ -5,6 +5,7 @@ FROM ${image} ARG deb_version ARG deb_codename=bullseye ARG deb_release=unstable +ARG network=mainnet ARG deb_repo="http://packages.o1test.net" ARG deb_profile ARG deb_suffix @@ -15,7 +16,7 @@ ENV SUFFIX=${deb_suffix:+-${deb_suffix}} # possible values: # - mina-archive # - mina-archive-lightnet etc. -ENV MINA_DEB=mina-archive${SUFFIX} +ENV MINA_DEB=mina-archive-${network}${SUFFIX} ENV DEBIAN_FRONTEND noninteractive RUN echo "Building image with version $deb_codename $deb_release $deb_version" diff --git a/scripts/debian/build.sh b/scripts/debian/build.sh index ab5208dc7de8..4512fa297d56 100755 --- a/scripts/debian/build.sh +++ b/scripts/debian/build.sh @@ -23,7 +23,9 @@ if [ $# -eq 0 ] echo "No arguments supplied. Building all known debian packages" build_logproc_deb build_keypair_deb - build_archive_deb + build_archive_berkeley_deb + build_archive_devnet_deb + build_archive_mainnet_deb build_batch_txn_deb build_daemon_berkeley_deb build_daemon_mainnet_deb diff --git a/scripts/debian/builder-helpers.sh b/scripts/debian/builder-helpers.sh index a11a1aac9020..866fc1470990 100755 --- a/scripts/debian/builder-helpers.sh +++ b/scripts/debian/builder-helpers.sh @@ -360,16 +360,8 @@ build_daemon_berkeley_deb() { } ##################################### END BERKELEY PACKAGE ###################################### -##################################### ARCHIVE PACKAGE ########################################### -build_archive_deb () { - ARCHIVE_DEB=mina-archive${DEB_SUFFIX} - - echo "------------------------------------------------------------" - echo "--- Building archive deb" - - - create_control_file "$ARCHIVE_DEB" "${ARCHIVE_DEPS}" 'Mina Archive Process - Compatible with Mina Daemon' +copy_common_archive_configs() { + local ARCHIVE_DEB="${1}" cp ./default/src/app/archive/archive.exe "${BUILDDIR}/usr/local/bin/mina-archive" cp ./default/src/app/archive_blocks/archive_blocks.exe "${BUILDDIR}/usr/local/bin/mina-archive-blocks" @@ -385,10 +377,54 @@ build_archive_deb () { cp ../src/app/archive/drop_tables.sql "${BUILDDIR}/etc/mina/archive" build_deb "$ARCHIVE_DEB" +} + +##################################### ARCHIVE DEVNET PACKAGE ########################################### +build_archive_devnet_deb () { + ARCHIVE_DEB=mina-archive-devnet + + echo "------------------------------------------------------------" + echo "--- Building archive devnet deb" + + create_control_file "$ARCHIVE_DEB" "${ARCHIVE_DEPS}" 'Mina Archive Process + Compatible with Mina Daemon' + + copy_common_archive_configs "$ARCHIVE_DEB" + +} +##################################### END ARCHIVE DEVNET PACKAGE ######################################## + +##################################### ARCHIVE BERKELEY PACKAGE ########################################### +build_archive_berkeley_deb () { + ARCHIVE_DEB=mina-archive${DEB_SUFFIX} + + echo "------------------------------------------------------------" + echo "--- Building archive berkeley deb" + + + create_control_file "$ARCHIVE_DEB" "${ARCHIVE_DEPS}" 'Mina Archive Process + Compatible with Mina Daemon' + + copy_common_archive_configs "$ARCHIVE_DEB" } ##################################### END ARCHIVE PACKAGE ######################################## +##################################### ARCHIVE MAINNET PACKAGE ########################################### +build_archive_mainnet_deb () { + ARCHIVE_DEB=mina-archive-mainnet + + echo "------------------------------------------------------------" + echo "--- Building archive mainnet deb" + + create_control_file "$ARCHIVE_DEB" "${ARCHIVE_DEPS}" 'Mina Archive Process + Compatible with Mina Daemon' + + copy_common_archive_configs "$ARCHIVE_DEB" + +} +##################################### END ARCHIVE MAINNET PACKAGE ######################################## + ##################################### ZKAPP TEST TXN ############################################# build_zkapp_test_transaction_deb () { From 304a4be2c44d1ba92fa06b0ab295df0af57792b9 Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 9 May 2025 16:23:01 +0200 Subject: [PATCH 04/27] add network suffix to archive debian --- buildkite/src/Constants/Artifacts.dhall | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildkite/src/Constants/Artifacts.dhall b/buildkite/src/Constants/Artifacts.dhall index 669dc75958a6..99bf353a9f43 100644 --- a/buildkite/src/Constants/Artifacts.dhall +++ b/buildkite/src/Constants/Artifacts.dhall @@ -88,7 +88,7 @@ let toDebianName = -> merge { Daemon = "daemon_${Network.lowerName network}" , LogProc = "logproc" - , Archive = "archive" + , Archive = "archive_${Network.lowerName network}" , TestExecutive = "test_executive" , BatchTxn = "batch_txn" , Rosetta = "rosetta_${Network.lowerName network}" @@ -108,7 +108,7 @@ let toDebianNames = ( \(a : Artifact) -> merge { Daemon = [ toDebianName a network ] - , Archive = [ "archive" ] + , Archive = [ toDebianName a network ] , LogProc = [ "logproc" ] , TestExecutive = [ "test_executive" ] , BatchTxn = [ "batch_txn" ] From b5934aa0a231f5c52b48f9f9f18df33fac83da93 Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 9 May 2025 21:20:38 +0200 Subject: [PATCH 05/27] add suffixes only for berkeley --- scripts/release-docker.sh | 37 ++++++++++++------------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/scripts/release-docker.sh b/scripts/release-docker.sh index 658b0f07f919..41a8438346a1 100755 --- a/scripts/release-docker.sh +++ b/scripts/release-docker.sh @@ -68,32 +68,19 @@ IMAGE="--build-arg image=${IMAGE}" # - hardfork # - lightnet # - hardfork-instrumented - case "${DEB_PROFILE}" in - standard) - case "${DEB_BUILD_FLAGS}" in - *instrumented) - DOCKER_DEB_SUFFIX="--build-arg deb_suffix=instrumented" - BUILD_FLAG_SUFFIX="-instrumented" - ;; - *) - ;; - esac - ;; - *) - case "${DEB_BUILD_FLAGS}" in - *instrumented) - DOCKER_DEB_SUFFIX="--build-arg deb_suffix=${DEB_PROFILE}-instrumented" - BUILD_FLAG_SUFFIX="-instrumented" - DEB_PROFILE_SUFFIX="-${DEB_PROFILE}" - ;; - *) - DOCKER_DEB_SUFFIX="--build-arg deb_suffix=${DEB_PROFILE}" - DEB_PROFILE_SUFFIX="-${DEB_PROFILE}" - ;; - esac - ;; - esac +if [[ ${NETWORK} == *"berkeley"* ]]; then + if [[ ${DEB_PROFILE} == *"lightnet"* ]]; then + DOCKER_DEB_SUFFIX="--build-arg deb_suffix=instrumented" + BUILD_FLAG_SUFFIX="-instrumented" + fi + + if [[ ${DEB_BUILD_FLAGS} == *"instrumented"* ]]; then + DOCKER_DEB_SUFFIX="--build-arg deb_suffix=instrumented" + BUILD_FLAG_SUFFIX="-instrumented" + fi +fi + # Debug prints for visability # Substring removal to cut the --build-arg arguments on the = so that the output is exactly the input flags https://wiki.bash-hackers.org/syntax/pe#substring_removal echo "--service ${SERVICE} --version ${VERSION} --branch ${BRANCH##*=} --deb-version ${DEB_VERSION##*=} --deb-suffix ${DOCKER_DEB_SUFFIX##*=} --deb-release ${DEB_RELEASE##*=} --deb-codename ${DEB_CODENAME##*=}" From bed8f053a89351c9d58ee41910d116a7be044893 Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 9 May 2025 21:22:42 +0200 Subject: [PATCH 06/27] some more comments --- scripts/release-docker.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/release-docker.sh b/scripts/release-docker.sh index 41a8438346a1..dd1afaa6e292 100755 --- a/scripts/release-docker.sh +++ b/scripts/release-docker.sh @@ -62,13 +62,13 @@ case "${DEB_CODENAME##*=}" in esac IMAGE="--build-arg image=${IMAGE}" -# Determine suffix for mina name. Suffix is combined from profile and service name +# Determine suffix for mina name. Suffix is combined from network or build flags/profile +# but only for non standard builds. # Possible outcomes: -# - instrumented -# - hardfork -# - lightnet -# - hardfork-instrumented - +# mina-berkeley-instrumented +# mina-archive-berkeley-instrumented +# mina-devnet +# mina-mainnet if [[ ${NETWORK} == *"berkeley"* ]]; then if [[ ${DEB_PROFILE} == *"lightnet"* ]]; then DOCKER_DEB_SUFFIX="--build-arg deb_suffix=instrumented" From 8facd5fb8dc4d039bafb7584f06dafd6ca8dbf84 Mon Sep 17 00:00:00 2001 From: dkijania Date: Sat, 10 May 2025 13:54:34 +0200 Subject: [PATCH 07/27] fix suffixes --- scripts/debian/builder-helpers.sh | 11 ++++------- scripts/release-docker.sh | 3 +-- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/scripts/debian/builder-helpers.sh b/scripts/debian/builder-helpers.sh index 866fc1470990..3b21274678c2 100755 --- a/scripts/debian/builder-helpers.sh +++ b/scripts/debian/builder-helpers.sh @@ -43,16 +43,13 @@ case "${MINA_DEB_CODENAME}" in ;; esac +MINA_DEB_NAME="mina-berkeley" + # Add suffix to debian to distinguish different profiles (mainnet/devnet/lightnet) case "${DUNE_PROFILE}" in - devnet|mainnet) - MINA_DEB_NAME="mina-berkeley" - DEB_SUFFIX="" - ;; - *) + lightnet) # use dune profile as suffix but replace underscore to dashes so deb builder won't complain _SUFFIX=${DUNE_PROFILE//_/-} - MINA_DEB_NAME="mina-berkeley-${_SUFFIX}" DEB_SUFFIX="-${_SUFFIX}" ;; esac @@ -396,7 +393,7 @@ build_archive_devnet_deb () { ##################################### ARCHIVE BERKELEY PACKAGE ########################################### build_archive_berkeley_deb () { - ARCHIVE_DEB=mina-archive${DEB_SUFFIX} + ARCHIVE_DEB=mina-archive-berkeley${DEB_SUFFIX} echo "------------------------------------------------------------" echo "--- Building archive berkeley deb" diff --git a/scripts/release-docker.sh b/scripts/release-docker.sh index dd1afaa6e292..6fcff64f1512 100755 --- a/scripts/release-docker.sh +++ b/scripts/release-docker.sh @@ -71,8 +71,7 @@ IMAGE="--build-arg image=${IMAGE}" # mina-mainnet if [[ ${NETWORK} == *"berkeley"* ]]; then if [[ ${DEB_PROFILE} == *"lightnet"* ]]; then - DOCKER_DEB_SUFFIX="--build-arg deb_suffix=instrumented" - BUILD_FLAG_SUFFIX="-instrumented" + DOCKER_DEB_SUFFIX="--build-arg deb_suffix=lightnet" fi if [[ ${DEB_BUILD_FLAGS} == *"instrumented"* ]]; then From a66de02740e109e653b2423c9de5ece5478c9c34 Mon Sep 17 00:00:00 2001 From: Dariusz Kijania Date: Sat, 10 May 2025 14:51:02 +0200 Subject: [PATCH 08/27] Update builder-helpers.sh --- scripts/debian/builder-helpers.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/debian/builder-helpers.sh b/scripts/debian/builder-helpers.sh index 3b21274678c2..b7e739d194dc 100755 --- a/scripts/debian/builder-helpers.sh +++ b/scripts/debian/builder-helpers.sh @@ -44,6 +44,7 @@ case "${MINA_DEB_CODENAME}" in esac MINA_DEB_NAME="mina-berkeley" +DEB_SUFFIX="" # Add suffix to debian to distinguish different profiles (mainnet/devnet/lightnet) case "${DUNE_PROFILE}" in From 72fec8001f870e32a8cc41419a50126acb4c9b0f Mon Sep 17 00:00:00 2001 From: Dariusz Kijania Date: Sat, 10 May 2025 14:55:34 +0200 Subject: [PATCH 09/27] Update helper.sh --- scripts/docker/helper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/docker/helper.sh b/scripts/docker/helper.sh index 0e85d32f4c43..0f6f602c752c 100644 --- a/scripts/docker/helper.sh +++ b/scripts/docker/helper.sh @@ -31,7 +31,7 @@ function export_suffixes () { # - lightnet # - hardfork-instrumented case "${DEB_PROFILE}" in - standard) + devnet) case "${DEB_BUILD_FLAGS}" in *instrumented) export DOCKER_DEB_SUFFIX="--build-arg deb_suffix=instrumented" From c1a93f96e87d3e9eaf6c02cb3558ad0978135bc1 Mon Sep 17 00:00:00 2001 From: Dariusz Kijania Date: Sat, 10 May 2025 16:24:59 +0200 Subject: [PATCH 10/27] Update Dockerfile-mina-rosetta --- dockerfiles/Dockerfile-mina-rosetta | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/Dockerfile-mina-rosetta b/dockerfiles/Dockerfile-mina-rosetta index 4741e4317b2f..d5d77c5217c2 100644 --- a/dockerfiles/Dockerfile-mina-rosetta +++ b/dockerfiles/Dockerfile-mina-rosetta @@ -91,7 +91,7 @@ RUN mkdir -p --mode=700 ${MINA_CONFIG_DIR}/wallets/store/ \ RUN echo "Building image with version $deb_version from repo $deb_release $deb_codename for network $network" \ && echo "deb [trusted=yes] ${deb_repo} $deb_codename $deb_release" > /etc/apt/sources.list.d/o1.list \ && apt-get update --quiet --yes \ - && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" "mina-rosetta-$network=$deb_version" "mina-archive=$deb_version" "mina-zkapp-test-transaction=$deb_version" \ + && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" "mina-rosetta-$network=$deb_version" "mina-archive_$network=$deb_version" "mina-zkapp-test-transaction=$deb_version" \ && rm -rf /var/lib/apt/lists/* # --- Set up postgres From 64698bdbbd2ad481aec5c75d1138a47ebfe57aaf Mon Sep 17 00:00:00 2001 From: Dariusz Kijania Date: Sat, 10 May 2025 17:22:01 +0200 Subject: [PATCH 11/27] Update MinaArtifact.dhall --- buildkite/src/Command/MinaArtifact.dhall | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 89aaa7d08aa7..c0015d12058e 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -1,4 +1,4 @@ -let Prelude = ../External/Prelude.dhall +9let Prelude = ../External/Prelude.dhall let List/map = Prelude.List.map @@ -206,6 +206,7 @@ let docker_step , service = Artifacts.Type.Archive , deb_codename = spec.debVersion , deb_profile = spec.profile + , network = Network.lowerName spec.network , build_flags = spec.buildFlags , docker_publish = docker_publish , deb_repo = DebianRepo.Type.Local From 7f4b5712fe38fb626edd87c6579b0e64b0020984 Mon Sep 17 00:00:00 2001 From: Dariusz Kijania Date: Sat, 10 May 2025 18:00:55 +0200 Subject: [PATCH 12/27] Update MinaArtifact.dhall --- buildkite/src/Command/MinaArtifact.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index c0015d12058e..868989e0fa67 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -1,4 +1,4 @@ -9let Prelude = ../External/Prelude.dhall +let Prelude = ../External/Prelude.dhall let List/map = Prelude.List.map From 2655b1cdd9b50700899bfc823b5fae632ff36375 Mon Sep 17 00:00:00 2001 From: Dariusz Kijania Date: Sat, 10 May 2025 18:10:56 +0200 Subject: [PATCH 13/27] Update Dockerfile-mina-rosetta --- dockerfiles/Dockerfile-mina-rosetta | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/Dockerfile-mina-rosetta b/dockerfiles/Dockerfile-mina-rosetta index d5d77c5217c2..ce19974124bc 100644 --- a/dockerfiles/Dockerfile-mina-rosetta +++ b/dockerfiles/Dockerfile-mina-rosetta @@ -91,7 +91,7 @@ RUN mkdir -p --mode=700 ${MINA_CONFIG_DIR}/wallets/store/ \ RUN echo "Building image with version $deb_version from repo $deb_release $deb_codename for network $network" \ && echo "deb [trusted=yes] ${deb_repo} $deb_codename $deb_release" > /etc/apt/sources.list.d/o1.list \ && apt-get update --quiet --yes \ - && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" "mina-rosetta-$network=$deb_version" "mina-archive_$network=$deb_version" "mina-zkapp-test-transaction=$deb_version" \ + && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" "mina-rosetta-$network=$deb_version" "mina-archive-$network=$deb_version" "mina-zkapp-test-transaction=$deb_version" \ && rm -rf /var/lib/apt/lists/* # --- Set up postgres From 6999cf3046d88cd99b5466ed9c117a566e0bd339 Mon Sep 17 00:00:00 2001 From: dkijania Date: Sat, 10 May 2025 20:16:13 +0200 Subject: [PATCH 14/27] fix suffix for archive / daemon dockers --- dockerfiles/Dockerfile-mina-test-suite | 2 +- scripts/debian/builder-helpers.sh | 1 + scripts/docker/helper.sh | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dockerfiles/Dockerfile-mina-test-suite b/dockerfiles/Dockerfile-mina-test-suite index 1162bfa5e9e0..065022ab825a 100644 --- a/dockerfiles/Dockerfile-mina-test-suite +++ b/dockerfiles/Dockerfile-mina-test-suite @@ -65,7 +65,7 @@ RUN locale-gen en_US.UTF-8 RUN echo "Building image with version $deb_version from repo $deb_release $deb_codename " \ && echo "deb [trusted=yes] ${deb_repo} $deb_codename $deb_release" > /etc/apt/sources.list.d/o1.list \ && apt-get update --quiet --yes \ - && apt-get install --quiet --yes --allow-downgrades -o Dpkg::Options::="--force-overwrite" "mina-test-suite=$deb_version" "mina-$network-instrumented=$deb_version" "mina-archive-instrumented=$deb_version" \ + && apt-get install --quiet --yes --allow-downgrades -o Dpkg::Options::="--force-overwrite" "mina-test-suite=$deb_version" "mina-$network-instrumented=$deb_version" "mina-archive-$network-instrumented=$deb_version" \ && rm -rf /var/lib/apt/lists/* ENTRYPOINT ["sleep","infinity"] \ No newline at end of file diff --git a/scripts/debian/builder-helpers.sh b/scripts/debian/builder-helpers.sh index b7e739d194dc..0574a9cc9821 100755 --- a/scripts/debian/builder-helpers.sh +++ b/scripts/debian/builder-helpers.sh @@ -52,6 +52,7 @@ case "${DUNE_PROFILE}" in # use dune profile as suffix but replace underscore to dashes so deb builder won't complain _SUFFIX=${DUNE_PROFILE//_/-} DEB_SUFFIX="-${_SUFFIX}" + MINA_DEB_NAME="${MINA_DEB_NAME}-${DEB_SUFFIX}" ;; esac diff --git a/scripts/docker/helper.sh b/scripts/docker/helper.sh index 0f6f602c752c..da6844f5f627 100644 --- a/scripts/docker/helper.sh +++ b/scripts/docker/helper.sh @@ -31,7 +31,7 @@ function export_suffixes () { # - lightnet # - hardfork-instrumented case "${DEB_PROFILE}" in - devnet) + devnet|mainnet) case "${DEB_BUILD_FLAGS}" in *instrumented) export DOCKER_DEB_SUFFIX="--build-arg deb_suffix=instrumented" From 29d2dd575281a2a0e1fe390252897936f12e0a02 Mon Sep 17 00:00:00 2001 From: dkijania Date: Sat, 10 May 2025 22:51:47 +0200 Subject: [PATCH 15/27] last fix for dependencies --- .../src/Jobs/Release/MinaArtifactBullseyeBerkeleyDevnet.dhall | 1 - .../MinaArtifactBullseyeBerkeleyDevnetInstrumented.dhall | 1 + scripts/debian/builder-helpers.sh | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeleyDevnet.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeleyDevnet.dhall index d4c2688f7218..4e6d1ddf031e 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeleyDevnet.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeleyDevnet.dhall @@ -17,7 +17,6 @@ in Pipeline.build , Artifacts.Type.TestExecutive , Artifacts.Type.Rosetta , Artifacts.Type.ZkappTestTransaction - , Artifacts.Type.FunctionalTestSuite ] , tags = [ PipelineTag.Type.Long diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeleyDevnetInstrumented.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeleyDevnetInstrumented.dhall index b36d1324dc3b..d6427849bb3f 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeleyDevnetInstrumented.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeleyDevnetInstrumented.dhall @@ -13,6 +13,7 @@ in Pipeline.build [ Artifacts.Type.Daemon , Artifacts.Type.LogProc , Artifacts.Type.Archive + , Artifacts.Type.FunctionalTestSuite ] , buildFlags = BuildFlags.Type.Instrumented } diff --git a/scripts/debian/builder-helpers.sh b/scripts/debian/builder-helpers.sh index 0574a9cc9821..3869931f9c63 100755 --- a/scripts/debian/builder-helpers.sh +++ b/scripts/debian/builder-helpers.sh @@ -51,7 +51,7 @@ case "${DUNE_PROFILE}" in lightnet) # use dune profile as suffix but replace underscore to dashes so deb builder won't complain _SUFFIX=${DUNE_PROFILE//_/-} - DEB_SUFFIX="-${_SUFFIX}" + DEB_SUFFIX="${_SUFFIX}" MINA_DEB_NAME="${MINA_DEB_NAME}-${DEB_SUFFIX}" ;; esac From 25ae7bdd2750c22001a066a86094b44a16229819 Mon Sep 17 00:00:00 2001 From: dkijania Date: Sat, 10 May 2025 23:11:31 +0200 Subject: [PATCH 16/27] fix instrumented jobs deps --- buildkite/src/Command/DockerImage.dhall | 4 +--- buildkite/src/Command/MinaArtifact.dhall | 6 ------ buildkite/src/Constants/DockerVersions.dhall | 21 ++++++++++++++----- .../src/Jobs/Test/PatchArchiveTest.dhall | 6 +++++- buildkite/src/Jobs/Test/ReplayerTest.dhall | 6 +++++- 5 files changed, 27 insertions(+), 16 deletions(-) diff --git a/buildkite/src/Command/DockerImage.dhall b/buildkite/src/Command/DockerImage.dhall index 47bbda439bdd..9c4d76fe3f49 100644 --- a/buildkite/src/Command/DockerImage.dhall +++ b/buildkite/src/Command/DockerImage.dhall @@ -69,9 +69,7 @@ let ReleaseSpec = let stepKey = \(spec : ReleaseSpec.Type) - -> "${Artifacts.lowerName - spec.service}${BuildFlags.toLabelSegment - spec.build_flags}${spec.step_key_suffix}" + -> "${Artifacts.lowerName spec.service}${spec.step_key_suffix}" let stepLabel = \(spec : ReleaseSpec.Type) diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index 868989e0fa67..70e57347462f 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -83,11 +83,6 @@ let nameSuffix spec.profile}${BuildFlags.toSuffixUppercase spec.buildFlags}" -let keySuffix - : MinaBuildSpec.Type -> Text - = \(spec : MinaBuildSpec.Type) - -> "${BuildFlags.toLabelSegment spec.buildFlags}" - let build_artifacts : MinaBuildSpec.Type -> Command.Type = \(spec : MinaBuildSpec.Type) @@ -322,5 +317,4 @@ in { pipeline = pipeline , publishToDebian = publish_to_debian_repo , MinaBuildSpec = MinaBuildSpec , labelSuffix = labelSuffix - , keySuffix = keySuffix } diff --git a/buildkite/src/Constants/DockerVersions.dhall b/buildkite/src/Constants/DockerVersions.dhall index eb2dfeb9600a..9ab951645d6c 100644 --- a/buildkite/src/Constants/DockerVersions.dhall +++ b/buildkite/src/Constants/DockerVersions.dhall @@ -4,6 +4,8 @@ let Artifacts = ./Artifacts.dhall let Network = ./Network.dhall +let BuildFlags = ./BuildFlags.dhall + let Docker : Type = < Bookworm | Bullseye | Jammy | Focal > @@ -33,9 +35,12 @@ let dependsOnStep = -> \(prefix : Text) -> \(network : Network.Type) -> \(profile : Profiles.Type) + -> \(buildFlag : BuildFlags.Type) -> \(binary : Artifacts.Type) -> let network = "${Network.capitalName network}" + let buildFlag = "${BuildFlags.toSuffixUppercase buildFlag}" + let profileSuffix = "${Profiles.toSuffixUppercase profile}" let suffix = "docker-image" @@ -46,14 +51,14 @@ let dependsOnStep = { Bookworm = [ { name = "${prefix}${capitalName - docker}${network}${profileSuffix}" + docker}${network}${profileSuffix}${buildFlag}" , key = key } ] , Bullseye = [ { name = "${prefix}${capitalName - docker}${network}${profileSuffix}" + docker}${network}${profileSuffix}${buildFlag}" , key = key } ] @@ -61,7 +66,7 @@ let dependsOnStep = [ { name = "${prefix}${capitalName docker}${network}${capitalName - docker}${profileSuffix}" + docker}${profileSuffix}${buildFlag}" , key = key } ] @@ -69,7 +74,7 @@ let dependsOnStep = [ { name = "${prefix}${capitalName docker}${network}${capitalName - docker}${profileSuffix}" + docker}${profileSuffix}${buildFlag}" , key = key } ] @@ -81,7 +86,13 @@ let dependsOn = -> \(network : Network.Type) -> \(profile : Profiles.Type) -> \(binary : Artifacts.Type) - -> dependsOnStep docker "MinaArtifact" network profile binary + -> dependsOnStep + docker + "MinaArtifact" + network + profile + BuildFlags.Type.None + binary in { Type = Docker , capitalName = capitalName diff --git a/buildkite/src/Jobs/Test/PatchArchiveTest.dhall b/buildkite/src/Jobs/Test/PatchArchiveTest.dhall index 8b3048ad6da7..edc3b896c328 100644 --- a/buildkite/src/Jobs/Test/PatchArchiveTest.dhall +++ b/buildkite/src/Jobs/Test/PatchArchiveTest.dhall @@ -10,6 +10,8 @@ let PatchArchiveTest = ../../Command/PatchArchiveTest.dhall let Profiles = ../../Constants/Profiles.dhall +let BuildFlags = ../../Constants/BuildFlags.dhall + let Network = ../../Constants/Network.dhall let Artifacts = ../../Constants/Artifacts.dhall @@ -17,10 +19,12 @@ let Artifacts = ../../Constants/Artifacts.dhall let Dockers = ../../Constants/DockerVersions.dhall let dependsOn = - Dockers.dependsOn + Dockers.dependsOnStep Dockers.Type.Bullseye + "MinaArtifact" Network.Type.Berkeley Profiles.Type.Devnet + BuildFlags.Type.Instrumented Artifacts.Type.FunctionalTestSuite in Pipeline.build diff --git a/buildkite/src/Jobs/Test/ReplayerTest.dhall b/buildkite/src/Jobs/Test/ReplayerTest.dhall index 717d6c4f3315..eeff1c15f866 100644 --- a/buildkite/src/Jobs/Test/ReplayerTest.dhall +++ b/buildkite/src/Jobs/Test/ReplayerTest.dhall @@ -10,6 +10,8 @@ let ReplayerTest = ../../Command/ReplayerTest.dhall let Profiles = ../../Constants/Profiles.dhall +let BuildFlags = ../../Constants/BuildFlags.dhall + let Dockers = ../../Constants/DockerVersions.dhall let Network = ../../Constants/Network.dhall @@ -17,10 +19,12 @@ let Network = ../../Constants/Network.dhall let Artifacts = ../../Constants/Artifacts.dhall let dependsOn = - Dockers.dependsOn + Dockers.dependsOnStep Dockers.Type.Bullseye + "MinaArtifact" Network.Type.Berkeley Profiles.Type.Devnet + BuildFlags.Type.Instrumented Artifacts.Type.FunctionalTestSuite in Pipeline.build From 56b4467e29a234e5c56a82617c6ab815fa61d43c Mon Sep 17 00:00:00 2001 From: "dariusz@o1labs.org" Date: Sun, 11 May 2025 19:50:00 +0200 Subject: [PATCH 17/27] fix deps --- buildkite/src/Command/Bench/Base.dhall | 6 +++++- buildkite/src/Command/PatchArchiveTest.dhall | 4 +++- buildkite/src/Command/ReplayerTest.dhall | 4 +++- buildkite/src/Command/RunWithPostgres.dhall | 7 +++---- buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall | 3 +-- 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/buildkite/src/Command/Bench/Base.dhall b/buildkite/src/Command/Bench/Base.dhall index 24a13303e4f9..9f04a5ce43a0 100644 --- a/buildkite/src/Command/Bench/Base.dhall +++ b/buildkite/src/Command/Bench/Base.dhall @@ -14,6 +14,8 @@ let RunInToolchain = ../../Command/RunInToolchain.dhall let Network = ../../Constants/Network.dhall +let BuildFlags = ../../Constants/BuildFlags.dhall + let Profiles = ../../Constants/Profiles.dhall let Command = ../../Command/Base.dhall @@ -49,10 +51,12 @@ let Spec = { mode = PipelineMode.Type.PullRequest , size = Size.Perf , dependsOn = - DebianVersions.dependsOn + DebianVersions.dependsOnStep DebianVersions.DebVersion.Bullseye + "MinaArtifact" Network.Type.Berkeley Profiles.Type.Devnet + BuildFlags.Type.Instrumented , additionalDirtyWhen = [] : List SelectFiles.Type , yellowThreshold = 0.1 , redThreshold = 0.2 diff --git a/buildkite/src/Command/PatchArchiveTest.dhall b/buildkite/src/Command/PatchArchiveTest.dhall index e220106c2361..3f5494f13898 100644 --- a/buildkite/src/Command/PatchArchiveTest.dhall +++ b/buildkite/src/Command/PatchArchiveTest.dhall @@ -6,6 +6,8 @@ let Size = ./Size.dhall let Network = ../Constants/Network.dhall +let BuildFlags = ../Constants/BuildFlags.dhall + let RunWithPostgres = ./RunWithPostgres.dhall let key = "patch-archive-test" @@ -20,7 +22,7 @@ in { step = , "NETWORK_DATA_FOLDER=/etc/mina/test/archive/sample_db" ] "./src/test/archive/sample_db/archive_db.sql" - Artifacts.Type.FunctionalTestSuite + "${Artifacts.name Artifacts.Type.FunctionalTestSuite}-${Network.lowerName network}-${BuildFlags.lowerName BuildFlags.Type.Instrumented}" (None Network.Type) "./scripts/patch-archive-test.sh && buildkite/scripts/upload-partial-coverage-data.sh ${key}" ] diff --git a/buildkite/src/Command/ReplayerTest.dhall b/buildkite/src/Command/ReplayerTest.dhall index 1e031edd4e27..b17b1d6c53ec 100644 --- a/buildkite/src/Command/ReplayerTest.dhall +++ b/buildkite/src/Command/ReplayerTest.dhall @@ -4,6 +4,8 @@ let Command = ./Base.dhall let Size = ./Size.dhall +let BuildFlags = ../Constants/BuildFlags.dhall + let RunWithPostgres = ./RunWithPostgres.dhall let Network = ../Constants/Network.dhall @@ -18,7 +20,7 @@ in { step = [ RunWithPostgres.runInDockerWithPostgresConn ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" - Artifacts.Type.FunctionalTestSuite + "${Artifacts.name Artifacts.Type.FunctionalTestSuite}-${Network.lowerName network}-${BuildFlags.lowerName BuildFlags.Type.Instrumented}" (None Network.Type) "./buildkite/scripts/replayer-test.sh && buildkite/scripts/upload-partial-coverage-data.sh ${key}" ] diff --git a/buildkite/src/Command/RunWithPostgres.dhall b/buildkite/src/Command/RunWithPostgres.dhall index b6279112a231..045380099c1e 100644 --- a/buildkite/src/Command/RunWithPostgres.dhall +++ b/buildkite/src/Command/RunWithPostgres.dhall @@ -19,13 +19,13 @@ let Network = ../Constants/Network.dhall let runInDockerWithPostgresConn : List Text -> Text - -> Artifacts.Type + -> Text -> Optional Network.Type -> Text -> Cmd.Type = \(environment : List Text) -> \(initScript : Text) - -> \(docker : Artifacts.Type) + -> \(docker : Text) -> \(network : Optional Network.Type) -> \(innerScript : Text) -> let port = "5432" @@ -79,8 +79,7 @@ let runInDockerWithPostgresConn , "docker run --network host --volume ${outerDir}:/workdir --workdir /workdir --name ${postgresDockerName} -d -e POSTGRES_USER=${user} -e POSTGRES_PASSWORD=${password} -e POSTGRES_PASSWORD=${password} -e POSTGRES_DB=${dbName} ${dockerVersion}" , "sleep 5" , "docker exec ${postgresDockerName} psql ${pg_conn} -f /workdir/${initScript}" - , "docker run --network host --volume ${outerDir}:/workdir --workdir /workdir --entrypoint bash ${envVars} gcr.io/o1labs-192920/${Artifacts.dockerName - docker}:${minaDockerTag}${networkOrDefault} ${innerScript}" + , "docker run --network host --volume ${outerDir}:/workdir --workdir /workdir --entrypoint bash ${envVars} gcr.io/o1labs-192920/${docker}:${minaDockerTag}${networkOrDefault} ${innerScript}" ] in { runInDockerWithPostgresConn = runInDockerWithPostgresConn } diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index 30e0a6a3f6f8..d981c4bb09d9 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -56,8 +56,7 @@ in Pipeline.build , RunWithPostgres.runInDockerWithPostgresConn ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" - Artifacts.Type.Rosetta - (Some network) + "${Artifacts.name Artifacts.Type.Rosetta}-${Network.lowerName network}" "./buildkite/scripts/rosetta-indexer-test.sh" , Cmd.runInDocker Cmd.Docker::{ From ee87fb1406b69e7bed8931d6f046b7f76b84ae8a Mon Sep 17 00:00:00 2001 From: Dariusz Kijania Date: Mon, 12 May 2025 11:31:57 +0200 Subject: [PATCH 18/27] Update Base.dhall --- buildkite/src/Command/Bench/Base.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Command/Bench/Base.dhall b/buildkite/src/Command/Bench/Base.dhall index 9f04a5ce43a0..5010fdd133c1 100644 --- a/buildkite/src/Command/Bench/Base.dhall +++ b/buildkite/src/Command/Bench/Base.dhall @@ -54,7 +54,7 @@ let Spec = DebianVersions.dependsOnStep DebianVersions.DebVersion.Bullseye "MinaArtifact" - Network.Type.Berkeley + (Some Network.Type.Berkeley) Profiles.Type.Devnet BuildFlags.Type.Instrumented , additionalDirtyWhen = [] : List SelectFiles.Type From 4d20733ab2c747e6558709cf1d8f53197d0e5abe Mon Sep 17 00:00:00 2001 From: Dariusz Kijania Date: Mon, 12 May 2025 11:34:58 +0200 Subject: [PATCH 19/27] Update PatchArchiveTest.dhall --- buildkite/src/Command/PatchArchiveTest.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Command/PatchArchiveTest.dhall b/buildkite/src/Command/PatchArchiveTest.dhall index 3f5494f13898..1ec45f55da09 100644 --- a/buildkite/src/Command/PatchArchiveTest.dhall +++ b/buildkite/src/Command/PatchArchiveTest.dhall @@ -22,7 +22,7 @@ in { step = , "NETWORK_DATA_FOLDER=/etc/mina/test/archive/sample_db" ] "./src/test/archive/sample_db/archive_db.sql" - "${Artifacts.name Artifacts.Type.FunctionalTestSuite}-${Network.lowerName network}-${BuildFlags.lowerName BuildFlags.Type.Instrumented}" + "${Artifacts.dockerName Artifacts.Type.FunctionalTestSuite}-${Network.lowerName network}-${BuildFlags.lowerName BuildFlags.Type.Instrumented}" (None Network.Type) "./scripts/patch-archive-test.sh && buildkite/scripts/upload-partial-coverage-data.sh ${key}" ] From 07d2bbf050b2ed99f563fd2c00a9bd7fb4820463 Mon Sep 17 00:00:00 2001 From: Dariusz Kijania Date: Mon, 12 May 2025 11:35:20 +0200 Subject: [PATCH 20/27] Update ReplayerTest.dhall --- buildkite/src/Command/ReplayerTest.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Command/ReplayerTest.dhall b/buildkite/src/Command/ReplayerTest.dhall index b17b1d6c53ec..bd5989cf0fa3 100644 --- a/buildkite/src/Command/ReplayerTest.dhall +++ b/buildkite/src/Command/ReplayerTest.dhall @@ -20,7 +20,7 @@ in { step = [ RunWithPostgres.runInDockerWithPostgresConn ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" - "${Artifacts.name Artifacts.Type.FunctionalTestSuite}-${Network.lowerName network}-${BuildFlags.lowerName BuildFlags.Type.Instrumented}" + "${Artifacts.dockerName Artifacts.Type.FunctionalTestSuite}-${Network.lowerName network}-${BuildFlags.lowerName BuildFlags.Type.Instrumented}" (None Network.Type) "./buildkite/scripts/replayer-test.sh && buildkite/scripts/upload-partial-coverage-data.sh ${key}" ] From 554ad689dac00868551f8802538d8a9e7ed0d305 Mon Sep 17 00:00:00 2001 From: "dariusz@o1labs.org" Date: Mon, 12 May 2025 19:59:15 +0200 Subject: [PATCH 21/27] fiz Base.dhall depends usage --- buildkite/src/Command/Bench/Base.dhall | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/buildkite/src/Command/Bench/Base.dhall b/buildkite/src/Command/Bench/Base.dhall index 5010fdd133c1..fce14e047ce2 100644 --- a/buildkite/src/Command/Bench/Base.dhall +++ b/buildkite/src/Command/Bench/Base.dhall @@ -52,11 +52,12 @@ let Spec = , size = Size.Perf , dependsOn = DebianVersions.dependsOnStep + (None Text) DebianVersions.DebVersion.Bullseye - "MinaArtifact" - (Some Network.Type.Berkeley) + Network.Type.Berkeley Profiles.Type.Devnet BuildFlags.Type.Instrumented + "build" , additionalDirtyWhen = [] : List SelectFiles.Type , yellowThreshold = 0.1 , redThreshold = 0.2 From 3730e2206ff6b2b276ae98e61118a35cb9d1c5aa Mon Sep 17 00:00:00 2001 From: "dariusz@o1labs.org" Date: Mon, 12 May 2025 20:07:09 +0200 Subject: [PATCH 22/27] fix network var --- buildkite/src/Command/PatchArchiveTest.dhall | 2 +- buildkite/src/Command/ReplayerTest.dhall | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildkite/src/Command/PatchArchiveTest.dhall b/buildkite/src/Command/PatchArchiveTest.dhall index 1ec45f55da09..46767d1ccc90 100644 --- a/buildkite/src/Command/PatchArchiveTest.dhall +++ b/buildkite/src/Command/PatchArchiveTest.dhall @@ -22,7 +22,7 @@ in { step = , "NETWORK_DATA_FOLDER=/etc/mina/test/archive/sample_db" ] "./src/test/archive/sample_db/archive_db.sql" - "${Artifacts.dockerName Artifacts.Type.FunctionalTestSuite}-${Network.lowerName network}-${BuildFlags.lowerName BuildFlags.Type.Instrumented}" + "${Artifacts.dockerName Artifacts.Type.FunctionalTestSuite}-${Network.lowerName Network.Type.Berkeley}-${BuildFlags.lowerName BuildFlags.Type.Instrumented}" (None Network.Type) "./scripts/patch-archive-test.sh && buildkite/scripts/upload-partial-coverage-data.sh ${key}" ] diff --git a/buildkite/src/Command/ReplayerTest.dhall b/buildkite/src/Command/ReplayerTest.dhall index bd5989cf0fa3..2869b8817212 100644 --- a/buildkite/src/Command/ReplayerTest.dhall +++ b/buildkite/src/Command/ReplayerTest.dhall @@ -20,7 +20,7 @@ in { step = [ RunWithPostgres.runInDockerWithPostgresConn ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" - "${Artifacts.dockerName Artifacts.Type.FunctionalTestSuite}-${Network.lowerName network}-${BuildFlags.lowerName BuildFlags.Type.Instrumented}" + "${Artifacts.dockerName Artifacts.Type.FunctionalTestSuite}-${Network.lowerName Network.Type.Berkeley}-${BuildFlags.lowerName BuildFlags.Type.Instrumented}" (None Network.Type) "./buildkite/scripts/replayer-test.sh && buildkite/scripts/upload-partial-coverage-data.sh ${key}" ] From 18a1846b128e74e118222f3a9662c8690b18a39f Mon Sep 17 00:00:00 2001 From: "dariusz@o1labs.org" Date: Mon, 12 May 2025 20:13:33 +0200 Subject: [PATCH 23/27] fix network in rosetta int tests --- buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index d981c4bb09d9..af60442843ae 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -56,7 +56,8 @@ in Pipeline.build , RunWithPostgres.runInDockerWithPostgresConn ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" - "${Artifacts.name Artifacts.Type.Rosetta}-${Network.lowerName network}" + "${Artifacts.name Artifacts.Type.Rosetta}" + (Some network) "./buildkite/scripts/rosetta-indexer-test.sh" , Cmd.runInDocker Cmd.Docker::{ From 0c0bb35c6558b8d72d0b9c71afe98ec53ca168ad Mon Sep 17 00:00:00 2001 From: "dariusz@o1labs.org" Date: Mon, 12 May 2025 20:21:10 +0200 Subject: [PATCH 24/27] name -> dockerName --- buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index af60442843ae..5e8e251e3950 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -56,7 +56,7 @@ in Pipeline.build , RunWithPostgres.runInDockerWithPostgresConn ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" - "${Artifacts.name Artifacts.Type.Rosetta}" + "${Artifacts.dockerName Artifacts.Type.Rosetta}" (Some network) "./buildkite/scripts/rosetta-indexer-test.sh" , Cmd.runInDocker From 8d54d5b9e0352ca1cf95b527429cd3dcad9a817c Mon Sep 17 00:00:00 2001 From: "dariusz@o1labs.org" Date: Tue, 13 May 2025 12:24:43 +0200 Subject: [PATCH 25/27] fix single node tests deps and mina-test-suite related deps --- buildkite/src/Command/PatchArchiveTest.dhall | 3 +-- buildkite/src/Command/ReplayerTest.dhall | 3 +-- buildkite/src/Command/RunWithPostgres.dhall | 17 +---------------- buildkite/src/Jobs/Test/SingleNodeTest.dhall | 7 ++++++- 4 files changed, 9 insertions(+), 21 deletions(-) diff --git a/buildkite/src/Command/PatchArchiveTest.dhall b/buildkite/src/Command/PatchArchiveTest.dhall index 46767d1ccc90..fa3ac81e8935 100644 --- a/buildkite/src/Command/PatchArchiveTest.dhall +++ b/buildkite/src/Command/PatchArchiveTest.dhall @@ -22,8 +22,7 @@ in { step = , "NETWORK_DATA_FOLDER=/etc/mina/test/archive/sample_db" ] "./src/test/archive/sample_db/archive_db.sql" - "${Artifacts.dockerName Artifacts.Type.FunctionalTestSuite}-${Network.lowerName Network.Type.Berkeley}-${BuildFlags.lowerName BuildFlags.Type.Instrumented}" - (None Network.Type) + "gcr.io/o1labs-192920/${Artifacts.dockerName Artifacts.Type.FunctionalTestSuite}:\\\$MINA_DOCKER_TAG-${Network.lowerName Network.Type.Berkeley}-${Buildflags.lowerName BuildFlags.Type.Instrumented}" "./scripts/patch-archive-test.sh && buildkite/scripts/upload-partial-coverage-data.sh ${key}" ] , label = "Archive: Patch Archive test" diff --git a/buildkite/src/Command/ReplayerTest.dhall b/buildkite/src/Command/ReplayerTest.dhall index 2869b8817212..d52010d21e72 100644 --- a/buildkite/src/Command/ReplayerTest.dhall +++ b/buildkite/src/Command/ReplayerTest.dhall @@ -20,8 +20,7 @@ in { step = [ RunWithPostgres.runInDockerWithPostgresConn ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" - "${Artifacts.dockerName Artifacts.Type.FunctionalTestSuite}-${Network.lowerName Network.Type.Berkeley}-${BuildFlags.lowerName BuildFlags.Type.Instrumented}" - (None Network.Type) + "gcr.io/o1labs-192920/${Artifacts.dockerName Artifacts.Type.FunctionalTestSuite}:\\\$MINA_DOCKER_TAG-${Network.lowerName Network.Type.Berkeley}-${Buildflags.lowerName BuildFlags.Type.Instrumented}" "./buildkite/scripts/replayer-test.sh && buildkite/scripts/upload-partial-coverage-data.sh ${key}" ] , label = "Archive: Replayer test" diff --git a/buildkite/src/Command/RunWithPostgres.dhall b/buildkite/src/Command/RunWithPostgres.dhall index 045380099c1e..f0f9cd6c57c6 100644 --- a/buildkite/src/Command/RunWithPostgres.dhall +++ b/buildkite/src/Command/RunWithPostgres.dhall @@ -20,13 +20,11 @@ let runInDockerWithPostgresConn : List Text -> Text -> Text - -> Optional Network.Type -> Text -> Cmd.Type = \(environment : List Text) -> \(initScript : Text) -> \(docker : Text) - -> \(network : Optional Network.Type) -> \(innerScript : Text) -> let port = "5432" @@ -60,26 +58,13 @@ let runInDockerWithPostgresConn : Text = "\\\$BUILDKITE_BUILD_CHECKOUT_PATH" - let minaDockerTag - : Text - = "\\\$MINA_DOCKER_TAG" - - let maybeNetwork = - Optional/map - Network.Type - Text - (\(network : Network.Type) -> "-${Network.lowerName network}") - network - - let networkOrDefault = Optional/default Text "" maybeNetwork - in Cmd.chain [ "( docker stop ${postgresDockerName} && docker rm ${postgresDockerName} ) || true" , "source buildkite/scripts/export-git-env-vars.sh" , "docker run --network host --volume ${outerDir}:/workdir --workdir /workdir --name ${postgresDockerName} -d -e POSTGRES_USER=${user} -e POSTGRES_PASSWORD=${password} -e POSTGRES_PASSWORD=${password} -e POSTGRES_DB=${dbName} ${dockerVersion}" , "sleep 5" , "docker exec ${postgresDockerName} psql ${pg_conn} -f /workdir/${initScript}" - , "docker run --network host --volume ${outerDir}:/workdir --workdir /workdir --entrypoint bash ${envVars} gcr.io/o1labs-192920/${docker}:${minaDockerTag}${networkOrDefault} ${innerScript}" + , "docker run --network host --volume ${outerDir}:/workdir --workdir /workdir --entrypoint bash ${envVars} ${docker} ${innerScript}" ] in { runInDockerWithPostgresConn = runInDockerWithPostgresConn } diff --git a/buildkite/src/Jobs/Test/SingleNodeTest.dhall b/buildkite/src/Jobs/Test/SingleNodeTest.dhall index 348aef0be5d7..3361e59618a2 100644 --- a/buildkite/src/Jobs/Test/SingleNodeTest.dhall +++ b/buildkite/src/Jobs/Test/SingleNodeTest.dhall @@ -16,6 +16,8 @@ let Profiles = ../../Constants/Profiles.dhall let Network = ../../Constants/Network.dhall +let BuildFlags = ../../Constants/BuildFlags.dhall + let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall @@ -25,10 +27,13 @@ let dependsOn = DebianVersions.DebVersion.Bullseye Network.Type.Berkeley Profiles.Type.Lightnet - # DebianVersions.dependsOn + # DebianVersions.dependsOnStep + (None Text) DebianVersions.DebVersion.Bullseye Network.Type.Berkeley Profiles.Type.Devnet + BuildFlags.Type.Instrumented + "build" let buildTestCmd : Size -> Command.Type From 9eaf8f70f99b5c7877a45da042943617fe1d7d3e Mon Sep 17 00:00:00 2001 From: "dariusz@o1labs.org" Date: Tue, 13 May 2025 12:28:31 +0200 Subject: [PATCH 26/27] fix imports --- buildkite/src/Command/PatchArchiveTest.dhall | 2 +- buildkite/src/Command/ReplayerTest.dhall | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildkite/src/Command/PatchArchiveTest.dhall b/buildkite/src/Command/PatchArchiveTest.dhall index fa3ac81e8935..387286fd496f 100644 --- a/buildkite/src/Command/PatchArchiveTest.dhall +++ b/buildkite/src/Command/PatchArchiveTest.dhall @@ -22,7 +22,7 @@ in { step = , "NETWORK_DATA_FOLDER=/etc/mina/test/archive/sample_db" ] "./src/test/archive/sample_db/archive_db.sql" - "gcr.io/o1labs-192920/${Artifacts.dockerName Artifacts.Type.FunctionalTestSuite}:\\\$MINA_DOCKER_TAG-${Network.lowerName Network.Type.Berkeley}-${Buildflags.lowerName BuildFlags.Type.Instrumented}" + "gcr.io/o1labs-192920/${Artifacts.dockerName Artifacts.Type.FunctionalTestSuite}:\\\$MINA_DOCKER_TAG-${Network.lowerName Network.Type.Berkeley}-${BuildFlags.lowerName BuildFlags.Type.Instrumented}" "./scripts/patch-archive-test.sh && buildkite/scripts/upload-partial-coverage-data.sh ${key}" ] , label = "Archive: Patch Archive test" diff --git a/buildkite/src/Command/ReplayerTest.dhall b/buildkite/src/Command/ReplayerTest.dhall index d52010d21e72..232221eb95c4 100644 --- a/buildkite/src/Command/ReplayerTest.dhall +++ b/buildkite/src/Command/ReplayerTest.dhall @@ -20,7 +20,7 @@ in { step = [ RunWithPostgres.runInDockerWithPostgresConn ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" - "gcr.io/o1labs-192920/${Artifacts.dockerName Artifacts.Type.FunctionalTestSuite}:\\\$MINA_DOCKER_TAG-${Network.lowerName Network.Type.Berkeley}-${Buildflags.lowerName BuildFlags.Type.Instrumented}" + "gcr.io/o1labs-192920/${Artifacts.dockerName Artifacts.Type.FunctionalTestSuite}:\\\$MINA_DOCKER_TAG-${Network.lowerName Network.Type.Berkeley}-${BuildFlags.lowerName BuildFlags.Type.Instrumented}" "./buildkite/scripts/replayer-test.sh && buildkite/scripts/upload-partial-coverage-data.sh ${key}" ] , label = "Archive: Replayer test" From 74f3149b6a9dde32abeafe2e48ad6d42e8d57b48 Mon Sep 17 00:00:00 2001 From: "dariusz@o1labs.org" Date: Tue, 13 May 2025 12:30:20 +0200 Subject: [PATCH 27/27] fix rosetta dep --- buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index 5e8e251e3950..02c78197de99 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -56,8 +56,7 @@ in Pipeline.build , RunWithPostgres.runInDockerWithPostgresConn ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" - "${Artifacts.dockerName Artifacts.Type.Rosetta}" - (Some network) + "gcr.io/o1labs-192920/${Artifacts.dockerName Artifacts.Type.Rosetta}:\\\$MINA_DOCKER_TAG-${Network.lowerName network}" "./buildkite/scripts/rosetta-indexer-test.sh" , Cmd.runInDocker Cmd.Docker::{