diff --git a/buildkite/src/Command/Bench/Base.dhall b/buildkite/src/Command/Bench/Base.dhall index c958a3b7b393..fce14e047ce2 100644 --- a/buildkite/src/Command/Bench/Base.dhall +++ b/buildkite/src/Command/Bench/Base.dhall @@ -14,6 +14,8 @@ let RunInToolchain = ../../Command/RunInToolchain.dhall let Network = ../../Constants/Network.dhall +let BuildFlags = ../../Constants/BuildFlags.dhall + let Profiles = ../../Constants/Profiles.dhall let Command = ../../Command/Base.dhall @@ -49,10 +51,13 @@ let Spec = { mode = PipelineMode.Type.PullRequest , size = Size.Perf , dependsOn = - DebianVersions.dependsOn + DebianVersions.dependsOnStep + (None Text) DebianVersions.DebVersion.Bullseye Network.Type.Berkeley - Profiles.Type.Standard + Profiles.Type.Devnet + BuildFlags.Type.Instrumented + "build" , additionalDirtyWhen = [] : List SelectFiles.Type , yellowThreshold = 0.1 , redThreshold = 0.2 diff --git a/buildkite/src/Command/Bench/LedgerApply.dhall b/buildkite/src/Command/Bench/LedgerApply.dhall index a38ef7c26f07..b6a01d0f189a 100644 --- a/buildkite/src/Command/Bench/LedgerApply.dhall +++ b/buildkite/src/Command/Bench/LedgerApply.dhall @@ -27,7 +27,7 @@ let dependsOn = (None Text) DebianVersions.DebVersion.Bullseye Network.Type.Berkeley - Profiles.Type.Standard + Profiles.Type.Devnet BuildFlags.Type.Instrumented "build" diff --git a/buildkite/src/Command/DockerImage.dhall b/buildkite/src/Command/DockerImage.dhall index dfd042260448..9c4d76fe3f49 100644 --- a/buildkite/src/Command/DockerImage.dhall +++ b/buildkite/src/Command/DockerImage.dhall @@ -56,7 +56,7 @@ let ReleaseSpec = , deb_codename = DebianVersions.DebVersion.Bullseye , deb_release = "\\\${MINA_DEB_RELEASE}" , deb_version = "\\\${MINA_DEB_VERSION}" - , deb_profile = Profiles.Type.Standard + , deb_profile = Profiles.Type.Devnet , build_flags = BuildFlags.Type.None , deb_repo = DebianRepo.Type.Local , docker_publish = DockerPublish.Type.Essential @@ -69,10 +69,7 @@ let ReleaseSpec = let stepKey = \(spec : ReleaseSpec.Type) - -> "${Artifacts.lowerName - spec.service}${Profiles.toLabelSegment - spec.deb_profile}${BuildFlags.toLabelSegment - spec.build_flags}${spec.step_key_suffix}" + -> "${Artifacts.lowerName spec.service}${spec.step_key_suffix}" let stepLabel = \(spec : ReleaseSpec.Type) diff --git a/buildkite/src/Command/HardforkPackageGeneration.dhall b/buildkite/src/Command/HardforkPackageGeneration.dhall index 12d9e8623740..12cdb70d2386 100644 --- a/buildkite/src/Command/HardforkPackageGeneration.dhall +++ b/buildkite/src/Command/HardforkPackageGeneration.dhall @@ -53,7 +53,7 @@ let Spec = , genesis_timestamp = Some "2024-04-07T11:45:00Z" , config_json_gz_url = "https://storage.googleapis.com/o1labs-gitops-infrastructure/devnet/devnet-state-dump-3NK4eDgbkCjKj9fFUXVkrJXsfpfXzJySoAvrFJVCropPW7LLF14F-676026c4d4d2c18a76b357d6422a06f932c3ef4667a8fd88717f68b53fd6b2d7.json.gz" - , profile = Profiles.Type.Standard + , profile = Profiles.Type.Devnet , suffix = "hardfork" } } diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall index d28ecbc4c18a..70e57347462f 100644 --- a/buildkite/src/Command/MinaArtifact.dhall +++ b/buildkite/src/Command/MinaArtifact.dhall @@ -54,7 +54,7 @@ let MinaBuildSpec = { prefix = "MinaArtifact" , artifacts = Artifacts.AllButTests , debVersion = DebianVersions.DebVersion.Bullseye - , profile = Profiles.Type.Standard + , profile = Profiles.Type.Devnet , buildFlags = BuildFlags.Type.None , network = Network.Type.Berkeley , toolchainSelectMode = Toolchain.SelectionMode.ByDebian @@ -83,12 +83,6 @@ let nameSuffix spec.profile}${BuildFlags.toSuffixUppercase spec.buildFlags}" -let keySuffix - : MinaBuildSpec.Type -> Text - = \(spec : MinaBuildSpec.Type) - -> "${Profiles.toLabelSegment spec.profile}${BuildFlags.toLabelSegment - spec.buildFlags}" - let build_artifacts : MinaBuildSpec.Type -> Command.Type = \(spec : MinaBuildSpec.Type) @@ -207,6 +201,7 @@ let docker_step , service = Artifacts.Type.Archive , deb_codename = spec.debVersion , deb_profile = spec.profile + , network = Network.lowerName spec.network , build_flags = spec.buildFlags , docker_publish = docker_publish , deb_repo = DebianRepo.Type.Local @@ -322,5 +317,4 @@ in { pipeline = pipeline , publishToDebian = publish_to_debian_repo , MinaBuildSpec = MinaBuildSpec , labelSuffix = labelSuffix - , keySuffix = keySuffix } diff --git a/buildkite/src/Command/PatchArchiveTest.dhall b/buildkite/src/Command/PatchArchiveTest.dhall index e220106c2361..387286fd496f 100644 --- a/buildkite/src/Command/PatchArchiveTest.dhall +++ b/buildkite/src/Command/PatchArchiveTest.dhall @@ -6,6 +6,8 @@ let Size = ./Size.dhall let Network = ../Constants/Network.dhall +let BuildFlags = ../Constants/BuildFlags.dhall + let RunWithPostgres = ./RunWithPostgres.dhall let key = "patch-archive-test" @@ -20,8 +22,7 @@ in { step = , "NETWORK_DATA_FOLDER=/etc/mina/test/archive/sample_db" ] "./src/test/archive/sample_db/archive_db.sql" - Artifacts.Type.FunctionalTestSuite - (None Network.Type) + "gcr.io/o1labs-192920/${Artifacts.dockerName Artifacts.Type.FunctionalTestSuite}:\\\$MINA_DOCKER_TAG-${Network.lowerName Network.Type.Berkeley}-${BuildFlags.lowerName BuildFlags.Type.Instrumented}" "./scripts/patch-archive-test.sh && buildkite/scripts/upload-partial-coverage-data.sh ${key}" ] , label = "Archive: Patch Archive test" diff --git a/buildkite/src/Command/Promotion/PromoteDebian.dhall b/buildkite/src/Command/Promotion/PromoteDebian.dhall index 66551ccfa6f7..9b4fb7dff747 100644 --- a/buildkite/src/Command/Promotion/PromoteDebian.dhall +++ b/buildkite/src/Command/Promotion/PromoteDebian.dhall @@ -50,7 +50,7 @@ let PromoteDebianSpec = , to_channel = DebianChannel.Type.Compatible , source_repo = DebianRepo.Type.Unstable , target_repo = DebianRepo.Type.Nightly - , profile = Profiles.Type.Standard + , profile = Profiles.Type.Devnet , remove_profile_from_name = False , step_key = "promote-debian-package" , if = None B/If @@ -68,7 +68,7 @@ let promoteDebianStep = then "--new-name ${Package.debianName spec.package - Profiles.Type.Standard + Profiles.Type.Devnet spec.network}" else "" diff --git a/buildkite/src/Command/Promotion/PromoteDocker.dhall b/buildkite/src/Command/Promotion/PromoteDocker.dhall index d2c19f88f4d4..4121f2e8f330 100644 --- a/buildkite/src/Command/Promotion/PromoteDocker.dhall +++ b/buildkite/src/Command/Promotion/PromoteDocker.dhall @@ -40,7 +40,7 @@ let PromoteDockerSpec = , version = "" , new_tags = [] : List Text , step_key = "promote-docker" - , profile = Profiles.Type.Standard + , profile = Profiles.Type.Devnet , network = Network.Type.Devnet , codename = DebianVersions.DebVersion.Bullseye , if = None B/If diff --git a/buildkite/src/Command/Promotion/PromotePackages.dhall b/buildkite/src/Command/Promotion/PromotePackages.dhall index 128d2b098403..d30878317315 100644 --- a/buildkite/src/Command/Promotion/PromotePackages.dhall +++ b/buildkite/src/Command/Promotion/PromotePackages.dhall @@ -70,7 +70,7 @@ let PromotePackagesSpec = , source_debian_repo = DebianRepo.Type.Unstable , target_debian_repo = DebianRepo.Type.Nightly , architecture = "amd64" - , profile = Profiles.Type.Standard + , profile = Profiles.Type.Devnet , network = Network.Type.Mainnet , codenames = [] : List DebianVersions.DebVersion , from_channel = DebianChannel.Type.Unstable diff --git a/buildkite/src/Command/Promotion/VerifyDebian.dhall b/buildkite/src/Command/Promotion/VerifyDebian.dhall index e178b8fbc694..df93895e0f6b 100644 --- a/buildkite/src/Command/Promotion/VerifyDebian.dhall +++ b/buildkite/src/Command/Promotion/VerifyDebian.dhall @@ -23,7 +23,7 @@ let promoteDebianVerificationStep = then "${Package.debianName spec.package - Profiles.Type.Standard + Profiles.Type.Devnet spec.network}" else Package.debianName spec.package spec.profile spec.network diff --git a/buildkite/src/Command/Promotion/VerifyPackages.dhall b/buildkite/src/Command/Promotion/VerifyPackages.dhall index c1010413f23c..f38797110c35 100644 --- a/buildkite/src/Command/Promotion/VerifyPackages.dhall +++ b/buildkite/src/Command/Promotion/VerifyPackages.dhall @@ -66,7 +66,7 @@ let VerifyPackagesSpec = , dockers = [] : List Artifact.Type , new_debian_version = "\\\\\$MINA_DEB_VERSION" , debian_repo = DebianRepo.Type.Local - , profile = Profiles.Type.Standard + , profile = Profiles.Type.Devnet , network = Network.Type.Mainnet , codenames = [] : List DebianVersions.DebVersion , channel = DebianChannel.Type.Compatible diff --git a/buildkite/src/Command/ReplayerTest.dhall b/buildkite/src/Command/ReplayerTest.dhall index 1e031edd4e27..232221eb95c4 100644 --- a/buildkite/src/Command/ReplayerTest.dhall +++ b/buildkite/src/Command/ReplayerTest.dhall @@ -4,6 +4,8 @@ let Command = ./Base.dhall let Size = ./Size.dhall +let BuildFlags = ../Constants/BuildFlags.dhall + let RunWithPostgres = ./RunWithPostgres.dhall let Network = ../Constants/Network.dhall @@ -18,8 +20,7 @@ in { step = [ RunWithPostgres.runInDockerWithPostgresConn ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" - Artifacts.Type.FunctionalTestSuite - (None Network.Type) + "gcr.io/o1labs-192920/${Artifacts.dockerName Artifacts.Type.FunctionalTestSuite}:\\\$MINA_DOCKER_TAG-${Network.lowerName Network.Type.Berkeley}-${BuildFlags.lowerName BuildFlags.Type.Instrumented}" "./buildkite/scripts/replayer-test.sh && buildkite/scripts/upload-partial-coverage-data.sh ${key}" ] , label = "Archive: Replayer test" diff --git a/buildkite/src/Command/Rosetta/Connectivity.dhall b/buildkite/src/Command/Rosetta/Connectivity.dhall index 0335aeeced16..00782b5f36c1 100644 --- a/buildkite/src/Command/Rosetta/Connectivity.dhall +++ b/buildkite/src/Command/Rosetta/Connectivity.dhall @@ -70,7 +70,7 @@ let command Dockers.dependsOn spec.dockerType spec.network - Profiles.Type.Standard + (Profiles.fromNetwork spec.network) Artifacts.Type.Rosetta } diff --git a/buildkite/src/Command/RunWithPostgres.dhall b/buildkite/src/Command/RunWithPostgres.dhall index b6279112a231..f0f9cd6c57c6 100644 --- a/buildkite/src/Command/RunWithPostgres.dhall +++ b/buildkite/src/Command/RunWithPostgres.dhall @@ -19,14 +19,12 @@ let Network = ../Constants/Network.dhall let runInDockerWithPostgresConn : List Text -> Text - -> Artifacts.Type - -> Optional Network.Type + -> Text -> Text -> Cmd.Type = \(environment : List Text) -> \(initScript : Text) - -> \(docker : Artifacts.Type) - -> \(network : Optional Network.Type) + -> \(docker : Text) -> \(innerScript : Text) -> let port = "5432" @@ -60,27 +58,13 @@ let runInDockerWithPostgresConn : Text = "\\\$BUILDKITE_BUILD_CHECKOUT_PATH" - let minaDockerTag - : Text - = "\\\$MINA_DOCKER_TAG" - - let maybeNetwork = - Optional/map - Network.Type - Text - (\(network : Network.Type) -> "-${Network.lowerName network}") - network - - let networkOrDefault = Optional/default Text "" maybeNetwork - in Cmd.chain [ "( docker stop ${postgresDockerName} && docker rm ${postgresDockerName} ) || true" , "source buildkite/scripts/export-git-env-vars.sh" , "docker run --network host --volume ${outerDir}:/workdir --workdir /workdir --name ${postgresDockerName} -d -e POSTGRES_USER=${user} -e POSTGRES_PASSWORD=${password} -e POSTGRES_PASSWORD=${password} -e POSTGRES_DB=${dbName} ${dockerVersion}" , "sleep 5" , "docker exec ${postgresDockerName} psql ${pg_conn} -f /workdir/${initScript}" - , "docker run --network host --volume ${outerDir}:/workdir --workdir /workdir --entrypoint bash ${envVars} gcr.io/o1labs-192920/${Artifacts.dockerName - docker}:${minaDockerTag}${networkOrDefault} ${innerScript}" + , "docker run --network host --volume ${outerDir}:/workdir --workdir /workdir --entrypoint bash ${envVars} ${docker} ${innerScript}" ] in { runInDockerWithPostgresConn = runInDockerWithPostgresConn } diff --git a/buildkite/src/Constants/Artifacts.dhall b/buildkite/src/Constants/Artifacts.dhall index 669dc75958a6..99bf353a9f43 100644 --- a/buildkite/src/Constants/Artifacts.dhall +++ b/buildkite/src/Constants/Artifacts.dhall @@ -88,7 +88,7 @@ let toDebianName = -> merge { Daemon = "daemon_${Network.lowerName network}" , LogProc = "logproc" - , Archive = "archive" + , Archive = "archive_${Network.lowerName network}" , TestExecutive = "test_executive" , BatchTxn = "batch_txn" , Rosetta = "rosetta_${Network.lowerName network}" @@ -108,7 +108,7 @@ let toDebianNames = ( \(a : Artifact) -> merge { Daemon = [ toDebianName a network ] - , Archive = [ "archive" ] + , Archive = [ toDebianName a network ] , LogProc = [ "logproc" ] , TestExecutive = [ "test_executive" ] , BatchTxn = [ "batch_txn" ] diff --git a/buildkite/src/Constants/DockerVersions.dhall b/buildkite/src/Constants/DockerVersions.dhall index eb2dfeb9600a..9ab951645d6c 100644 --- a/buildkite/src/Constants/DockerVersions.dhall +++ b/buildkite/src/Constants/DockerVersions.dhall @@ -4,6 +4,8 @@ let Artifacts = ./Artifacts.dhall let Network = ./Network.dhall +let BuildFlags = ./BuildFlags.dhall + let Docker : Type = < Bookworm | Bullseye | Jammy | Focal > @@ -33,9 +35,12 @@ let dependsOnStep = -> \(prefix : Text) -> \(network : Network.Type) -> \(profile : Profiles.Type) + -> \(buildFlag : BuildFlags.Type) -> \(binary : Artifacts.Type) -> let network = "${Network.capitalName network}" + let buildFlag = "${BuildFlags.toSuffixUppercase buildFlag}" + let profileSuffix = "${Profiles.toSuffixUppercase profile}" let suffix = "docker-image" @@ -46,14 +51,14 @@ let dependsOnStep = { Bookworm = [ { name = "${prefix}${capitalName - docker}${network}${profileSuffix}" + docker}${network}${profileSuffix}${buildFlag}" , key = key } ] , Bullseye = [ { name = "${prefix}${capitalName - docker}${network}${profileSuffix}" + docker}${network}${profileSuffix}${buildFlag}" , key = key } ] @@ -61,7 +66,7 @@ let dependsOnStep = [ { name = "${prefix}${capitalName docker}${network}${capitalName - docker}${profileSuffix}" + docker}${profileSuffix}${buildFlag}" , key = key } ] @@ -69,7 +74,7 @@ let dependsOnStep = [ { name = "${prefix}${capitalName docker}${network}${capitalName - docker}${profileSuffix}" + docker}${profileSuffix}${buildFlag}" , key = key } ] @@ -81,7 +86,13 @@ let dependsOn = -> \(network : Network.Type) -> \(profile : Profiles.Type) -> \(binary : Artifacts.Type) - -> dependsOnStep docker "MinaArtifact" network profile binary + -> dependsOnStep + docker + "MinaArtifact" + network + profile + BuildFlags.Type.None + binary in { Type = Docker , capitalName = capitalName diff --git a/buildkite/src/Constants/Profiles.dhall b/buildkite/src/Constants/Profiles.dhall index cf7cb3da71ac..54b42e5db308 100644 --- a/buildkite/src/Constants/Profiles.dhall +++ b/buildkite/src/Constants/Profiles.dhall @@ -1,11 +1,13 @@ +let Network = ./Network.dhall + let Profile : Type - = < Standard | Mainnet | Lightnet | Hardfork | Dev > + = < Devnet | Mainnet | Lightnet | Hardfork | Dev > let capitalName = \(profile : Profile) -> merge - { Standard = "Standard" + { Devnet = "Devnet" , Mainnet = "Mainnet" , Lightnet = "Lightnet" , Hardfork = "Hardfork" @@ -16,7 +18,7 @@ let capitalName = let lowerName = \(profile : Profile) -> merge - { Standard = "standard" + { Devnet = "devnet" , Mainnet = "mainnet" , Lightnet = "lightnet" , Hardfork = "hardfork" @@ -27,7 +29,7 @@ let lowerName = let duneProfile = \(profile : Profile) -> merge - { Standard = "devnet" + { Devnet = "devnet" , Mainnet = "mainnet" , Lightnet = "lightnet" , Hardfork = "hardfork" @@ -35,10 +37,19 @@ let duneProfile = } profile +let fromNetwork = + \(network : Network.Type) + -> merge + { Devnet = Profile.Devnet + , Mainnet = Profile.Mainnet + , Berkeley = Profile.Devnet + } + network + let toSuffixUppercase = \(profile : Profile) -> merge - { Standard = "" + { Devnet = "Devnet" , Mainnet = "Mainnet" , Lightnet = "Lightnet" , Hardfork = "Hardfork" @@ -49,7 +60,7 @@ let toSuffixUppercase = let toSuffixLowercase = \(profile : Profile) -> merge - { Standard = "" + { Devnet = "devnet" , Mainnet = "mainnet" , Lightnet = "lightnet" , Hardfork = "hardfork" @@ -60,7 +71,7 @@ let toSuffixLowercase = let toLabelSegment = \(profile : Profile) -> merge - { Standard = "" + { Devnet = "devnet" , Mainnet = "-mainnet" , Lightnet = "-lightnet" , Hardfork = "-hardfork" @@ -73,6 +84,7 @@ in { Type = Profile , lowerName = lowerName , duneProfile = duneProfile , toSuffixUppercase = toSuffixUppercase + , fromNetwork = fromNetwork , toSuffixLowercase = toSuffixLowercase , toLabelSegment = toLabelSegment } diff --git a/buildkite/src/Jobs/Promote/AutoPromoteNightly.dhall b/buildkite/src/Jobs/Promote/AutoPromoteNightly.dhall index 1ae82d7f0a9a..25ce2bc5798f 100644 --- a/buildkite/src/Jobs/Promote/AutoPromoteNightly.dhall +++ b/buildkite/src/Jobs/Promote/AutoPromoteNightly.dhall @@ -53,7 +53,7 @@ let promotePackages = , version = "\\\${FROM_VERSION_MANUAL:-\\\${MINA_DEB_VERSION}}" , architecture = "amd64" , new_debian_version = "\\\$(date \"+%Y%m%d\")" - , profile = Profiles.Type.Standard + , profile = Profiles.Type.Devnet , network = Network.Type.Devnet , codenames = [ DebianVersions.DebVersion.Bullseye, DebianVersions.DebVersion.Focal ] @@ -70,7 +70,7 @@ let verifyPackages = , debians = [] : List DebianPackage.Type , dockers = [ Artifacts.Type.Daemon, Artifacts.Type.Archive ] , new_debian_version = "\\\$(date \"+%Y%m%d\")" - , profile = Profiles.Type.Standard + , profile = Profiles.Type.Devnet , network = Network.Type.Devnet , codenames = [ DebianVersions.DebVersion.Bullseye, DebianVersions.DebVersion.Focal ] diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeley.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeleyDevnet.dhall similarity index 94% rename from buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeley.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeleyDevnet.dhall index d4c2688f7218..4e6d1ddf031e 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeley.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeleyDevnet.dhall @@ -17,7 +17,6 @@ in Pipeline.build , Artifacts.Type.TestExecutive , Artifacts.Type.Rosetta , Artifacts.Type.ZkappTestTransaction - , Artifacts.Type.FunctionalTestSuite ] , tags = [ PipelineTag.Type.Long diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeleyInstrumented.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeleyDevnetInstrumented.dhall similarity index 91% rename from buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeleyInstrumented.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeleyDevnetInstrumented.dhall index b36d1324dc3b..d6427849bb3f 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeleyInstrumented.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactBullseyeBerkeleyDevnetInstrumented.dhall @@ -13,6 +13,7 @@ in Pipeline.build [ Artifacts.Type.Daemon , Artifacts.Type.LogProc , Artifacts.Type.Archive + , Artifacts.Type.FunctionalTestSuite ] , buildFlags = BuildFlags.Type.Instrumented } diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnet.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnet.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeDevnetDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall b/buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnetMainnet.dhall similarity index 90% rename from buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall rename to buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnetMainnet.dhall index 97013973aca3..bc0b99315ea7 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnet.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactBullseyeMainnetMainnet.dhall @@ -4,6 +4,8 @@ let Network = ../../Constants/Network.dhall let Artifacts = ../../Constants/Artifacts.dhall +let Profiles = ../../Constants/Profiles.dhall + let Pipeline = ../../Pipeline/Dsl.dhall let PipelineMode = ../../Pipeline/Mode.dhall @@ -27,5 +29,6 @@ in Pipeline.build , PipelineTag.Type.Stable ] , mode = PipelineMode.Type.Stable + , profile = Profiles.Type.Mainnet } ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactFocalBerkeley.dhall b/buildkite/src/Jobs/Release/MinaArtifactFocalBerkeleyDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactFocalBerkeley.dhall rename to buildkite/src/Jobs/Release/MinaArtifactFocalBerkeleyDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactFocalDevnet.dhall b/buildkite/src/Jobs/Release/MinaArtifactFocalDevnetDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactFocalDevnet.dhall rename to buildkite/src/Jobs/Release/MinaArtifactFocalDevnetDevnet.dhall diff --git a/buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall b/buildkite/src/Jobs/Release/MinaArtifactFocalMainnetMainnet.dhall similarity index 91% rename from buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall rename to buildkite/src/Jobs/Release/MinaArtifactFocalMainnetMainnet.dhall index 8259ddc852ed..1b2dd6d90b82 100644 --- a/buildkite/src/Jobs/Release/MinaArtifactFocalMainnet.dhall +++ b/buildkite/src/Jobs/Release/MinaArtifactFocalMainnetMainnet.dhall @@ -6,6 +6,8 @@ let Network = ../../Constants/Network.dhall let Artifacts = ../../Constants/Artifacts.dhall +let Profiles = ../../Constants/Profiles.dhall + let Pipeline = ../../Pipeline/Dsl.dhall let PipelineMode = ../../Pipeline/Mode.dhall @@ -30,5 +32,6 @@ in Pipeline.build , PipelineTag.Type.Stable ] , mode = PipelineMode.Type.Stable + , profile = Profiles.Type.Mainnet } ) diff --git a/buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseyeBerkeley.dhall b/buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseyeBerkeleyDevnet.dhall similarity index 100% rename from buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseyeBerkeley.dhall rename to buildkite/src/Jobs/Release/MinaArtifactOnlyDebianBullseyeBerkeleyDevnet.dhall diff --git a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall index bffabd5a6537..5b27a6fe530e 100644 --- a/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall +++ b/buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall @@ -18,7 +18,7 @@ let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye Network.Type.Berkeley - Profiles.Type.Standard + Profiles.Type.Devnet in Pipeline.build Pipeline.Config::{ diff --git a/buildkite/src/Jobs/Test/ConnectToDevnet.dhall b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall index c7e913811f48..45846d73a28b 100644 --- a/buildkite/src/Jobs/Test/ConnectToDevnet.dhall +++ b/buildkite/src/Jobs/Test/ConnectToDevnet.dhall @@ -28,7 +28,7 @@ let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye network - Profiles.Type.Standard + Profiles.Type.Devnet Artifacts.Type.Daemon in Pipeline.build diff --git a/buildkite/src/Jobs/Test/ConnectToMainnet.dhall b/buildkite/src/Jobs/Test/ConnectToMainnet.dhall index a07829e9d8b7..7359ef59ff43 100644 --- a/buildkite/src/Jobs/Test/ConnectToMainnet.dhall +++ b/buildkite/src/Jobs/Test/ConnectToMainnet.dhall @@ -28,7 +28,7 @@ let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye network - Profiles.Type.Standard + Profiles.Type.Mainnet Artifacts.Type.Daemon in Pipeline.build diff --git a/buildkite/src/Jobs/Test/PatchArchiveTest.dhall b/buildkite/src/Jobs/Test/PatchArchiveTest.dhall index 6131cc2feb0e..edc3b896c328 100644 --- a/buildkite/src/Jobs/Test/PatchArchiveTest.dhall +++ b/buildkite/src/Jobs/Test/PatchArchiveTest.dhall @@ -10,6 +10,8 @@ let PatchArchiveTest = ../../Command/PatchArchiveTest.dhall let Profiles = ../../Constants/Profiles.dhall +let BuildFlags = ../../Constants/BuildFlags.dhall + let Network = ../../Constants/Network.dhall let Artifacts = ../../Constants/Artifacts.dhall @@ -17,10 +19,12 @@ let Artifacts = ../../Constants/Artifacts.dhall let Dockers = ../../Constants/DockerVersions.dhall let dependsOn = - Dockers.dependsOn + Dockers.dependsOnStep Dockers.Type.Bullseye + "MinaArtifact" Network.Type.Berkeley - Profiles.Type.Standard + Profiles.Type.Devnet + BuildFlags.Type.Instrumented Artifacts.Type.FunctionalTestSuite in Pipeline.build diff --git a/buildkite/src/Jobs/Test/ReplayerTest.dhall b/buildkite/src/Jobs/Test/ReplayerTest.dhall index 3986b8f57d99..eeff1c15f866 100644 --- a/buildkite/src/Jobs/Test/ReplayerTest.dhall +++ b/buildkite/src/Jobs/Test/ReplayerTest.dhall @@ -10,6 +10,8 @@ let ReplayerTest = ../../Command/ReplayerTest.dhall let Profiles = ../../Constants/Profiles.dhall +let BuildFlags = ../../Constants/BuildFlags.dhall + let Dockers = ../../Constants/DockerVersions.dhall let Network = ../../Constants/Network.dhall @@ -17,10 +19,12 @@ let Network = ../../Constants/Network.dhall let Artifacts = ../../Constants/Artifacts.dhall let dependsOn = - Dockers.dependsOn + Dockers.dependsOnStep Dockers.Type.Bullseye + "MinaArtifact" Network.Type.Berkeley - Profiles.Type.Standard + Profiles.Type.Devnet + BuildFlags.Type.Instrumented Artifacts.Type.FunctionalTestSuite in Pipeline.build diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index 70868cd62453..02c78197de99 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -56,8 +56,7 @@ in Pipeline.build , RunWithPostgres.runInDockerWithPostgresConn ([] : List Text) "./src/test/archive/sample_db/archive_db.sql" - Artifacts.Type.Rosetta - (Some network) + "gcr.io/o1labs-192920/${Artifacts.dockerName Artifacts.Type.Rosetta}:\\\$MINA_DOCKER_TAG-${Network.lowerName network}" "./buildkite/scripts/rosetta-indexer-test.sh" , Cmd.runInDocker Cmd.Docker::{ @@ -75,7 +74,7 @@ in Pipeline.build Dockers.dependsOn Dockers.Type.Bullseye network - Profiles.Type.Standard + Profiles.Type.Devnet Artifacts.Type.Rosetta } ] diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall index a67a439b17cb..a6e9179c01e8 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall @@ -71,7 +71,7 @@ in Pipeline.build Dockers.dependsOn Dockers.Type.Bullseye network - Profiles.Type.Standard + Profiles.Type.Devnet Artifacts.Type.Rosetta } ] diff --git a/buildkite/src/Jobs/Test/SingleNodeTest.dhall b/buildkite/src/Jobs/Test/SingleNodeTest.dhall index 9a695bf41981..3361e59618a2 100644 --- a/buildkite/src/Jobs/Test/SingleNodeTest.dhall +++ b/buildkite/src/Jobs/Test/SingleNodeTest.dhall @@ -16,6 +16,8 @@ let Profiles = ../../Constants/Profiles.dhall let Network = ../../Constants/Network.dhall +let BuildFlags = ../../Constants/BuildFlags.dhall + let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall @@ -25,10 +27,13 @@ let dependsOn = DebianVersions.DebVersion.Bullseye Network.Type.Berkeley Profiles.Type.Lightnet - # DebianVersions.dependsOn + # DebianVersions.dependsOnStep + (None Text) DebianVersions.DebVersion.Bullseye Network.Type.Berkeley - Profiles.Type.Standard + Profiles.Type.Devnet + BuildFlags.Type.Instrumented + "build" let buildTestCmd : Size -> Command.Type diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall index b03adbe28d7b..26fb8b600029 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall @@ -22,12 +22,12 @@ let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye Network.Type.Berkeley - Profiles.Type.Standard + Profiles.Type.Devnet Artifacts.Type.Daemon # Dockers.dependsOn Dockers.Type.Bullseye Network.Type.Berkeley - Profiles.Type.Standard + Profiles.Type.Devnet Artifacts.Type.Archive in Pipeline.build diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall index 420d92021a85..698d4df504af 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall @@ -22,12 +22,12 @@ let dependsOn = Dockers.dependsOn Dockers.Type.Bullseye Network.Type.Berkeley - Profiles.Type.Standard + Profiles.Type.Devnet Artifacts.Type.Daemon # Dockers.dependsOn Dockers.Type.Bullseye Network.Type.Berkeley - Profiles.Type.Standard + Profiles.Type.Devnet Artifacts.Type.Archive in Pipeline.build diff --git a/buildkite/src/Jobs/Test/VersionLint.dhall b/buildkite/src/Jobs/Test/VersionLint.dhall index 9ebc8a9f6253..0f795abe30b6 100644 --- a/buildkite/src/Jobs/Test/VersionLint.dhall +++ b/buildkite/src/Jobs/Test/VersionLint.dhall @@ -28,7 +28,7 @@ let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye Network.Type.Berkeley - Profiles.Type.Standard + Profiles.Type.Devnet let buildTestCmd : Text -> Size -> List Command.TaggedKey.Type -> B/SoftFail -> Command.Type diff --git a/dockerfiles/Dockerfile-mina-archive b/dockerfiles/Dockerfile-mina-archive index f340607992f6..20e9d7aa9b5f 100644 --- a/dockerfiles/Dockerfile-mina-archive +++ b/dockerfiles/Dockerfile-mina-archive @@ -5,6 +5,7 @@ FROM ${image} ARG deb_version ARG deb_codename=bullseye ARG deb_release=unstable +ARG network=mainnet ARG deb_repo="http://packages.o1test.net" ARG deb_profile ARG deb_suffix @@ -15,7 +16,7 @@ ENV SUFFIX=${deb_suffix:+-${deb_suffix}} # possible values: # - mina-archive # - mina-archive-lightnet etc. -ENV MINA_DEB=mina-archive${SUFFIX} +ENV MINA_DEB=mina-archive-${network}${SUFFIX} ENV DEBIAN_FRONTEND noninteractive RUN echo "Building image with version $deb_codename $deb_release $deb_version" diff --git a/dockerfiles/Dockerfile-mina-rosetta b/dockerfiles/Dockerfile-mina-rosetta index 4741e4317b2f..ce19974124bc 100644 --- a/dockerfiles/Dockerfile-mina-rosetta +++ b/dockerfiles/Dockerfile-mina-rosetta @@ -91,7 +91,7 @@ RUN mkdir -p --mode=700 ${MINA_CONFIG_DIR}/wallets/store/ \ RUN echo "Building image with version $deb_version from repo $deb_release $deb_codename for network $network" \ && echo "deb [trusted=yes] ${deb_repo} $deb_codename $deb_release" > /etc/apt/sources.list.d/o1.list \ && apt-get update --quiet --yes \ - && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" "mina-rosetta-$network=$deb_version" "mina-archive=$deb_version" "mina-zkapp-test-transaction=$deb_version" \ + && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" "mina-rosetta-$network=$deb_version" "mina-archive-$network=$deb_version" "mina-zkapp-test-transaction=$deb_version" \ && rm -rf /var/lib/apt/lists/* # --- Set up postgres diff --git a/dockerfiles/Dockerfile-mina-test-suite b/dockerfiles/Dockerfile-mina-test-suite index 1162bfa5e9e0..065022ab825a 100644 --- a/dockerfiles/Dockerfile-mina-test-suite +++ b/dockerfiles/Dockerfile-mina-test-suite @@ -65,7 +65,7 @@ RUN locale-gen en_US.UTF-8 RUN echo "Building image with version $deb_version from repo $deb_release $deb_codename " \ && echo "deb [trusted=yes] ${deb_repo} $deb_codename $deb_release" > /etc/apt/sources.list.d/o1.list \ && apt-get update --quiet --yes \ - && apt-get install --quiet --yes --allow-downgrades -o Dpkg::Options::="--force-overwrite" "mina-test-suite=$deb_version" "mina-$network-instrumented=$deb_version" "mina-archive-instrumented=$deb_version" \ + && apt-get install --quiet --yes --allow-downgrades -o Dpkg::Options::="--force-overwrite" "mina-test-suite=$deb_version" "mina-$network-instrumented=$deb_version" "mina-archive-$network-instrumented=$deb_version" \ && rm -rf /var/lib/apt/lists/* ENTRYPOINT ["sleep","infinity"] \ No newline at end of file diff --git a/scripts/debian/build.sh b/scripts/debian/build.sh index ab5208dc7de8..4512fa297d56 100755 --- a/scripts/debian/build.sh +++ b/scripts/debian/build.sh @@ -23,7 +23,9 @@ if [ $# -eq 0 ] echo "No arguments supplied. Building all known debian packages" build_logproc_deb build_keypair_deb - build_archive_deb + build_archive_berkeley_deb + build_archive_devnet_deb + build_archive_mainnet_deb build_batch_txn_deb build_daemon_berkeley_deb build_daemon_mainnet_deb diff --git a/scripts/debian/builder-helpers.sh b/scripts/debian/builder-helpers.sh index a11a1aac9020..3869931f9c63 100755 --- a/scripts/debian/builder-helpers.sh +++ b/scripts/debian/builder-helpers.sh @@ -43,17 +43,16 @@ case "${MINA_DEB_CODENAME}" in ;; esac +MINA_DEB_NAME="mina-berkeley" +DEB_SUFFIX="" + # Add suffix to debian to distinguish different profiles (mainnet/devnet/lightnet) case "${DUNE_PROFILE}" in - devnet|mainnet) - MINA_DEB_NAME="mina-berkeley" - DEB_SUFFIX="" - ;; - *) + lightnet) # use dune profile as suffix but replace underscore to dashes so deb builder won't complain _SUFFIX=${DUNE_PROFILE//_/-} - MINA_DEB_NAME="mina-berkeley-${_SUFFIX}" - DEB_SUFFIX="-${_SUFFIX}" + DEB_SUFFIX="${_SUFFIX}" + MINA_DEB_NAME="${MINA_DEB_NAME}-${DEB_SUFFIX}" ;; esac @@ -360,16 +359,8 @@ build_daemon_berkeley_deb() { } ##################################### END BERKELEY PACKAGE ###################################### -##################################### ARCHIVE PACKAGE ########################################### -build_archive_deb () { - ARCHIVE_DEB=mina-archive${DEB_SUFFIX} - - echo "------------------------------------------------------------" - echo "--- Building archive deb" - - - create_control_file "$ARCHIVE_DEB" "${ARCHIVE_DEPS}" 'Mina Archive Process - Compatible with Mina Daemon' +copy_common_archive_configs() { + local ARCHIVE_DEB="${1}" cp ./default/src/app/archive/archive.exe "${BUILDDIR}/usr/local/bin/mina-archive" cp ./default/src/app/archive_blocks/archive_blocks.exe "${BUILDDIR}/usr/local/bin/mina-archive-blocks" @@ -385,10 +376,54 @@ build_archive_deb () { cp ../src/app/archive/drop_tables.sql "${BUILDDIR}/etc/mina/archive" build_deb "$ARCHIVE_DEB" +} + +##################################### ARCHIVE DEVNET PACKAGE ########################################### +build_archive_devnet_deb () { + ARCHIVE_DEB=mina-archive-devnet + + echo "------------------------------------------------------------" + echo "--- Building archive devnet deb" + + create_control_file "$ARCHIVE_DEB" "${ARCHIVE_DEPS}" 'Mina Archive Process + Compatible with Mina Daemon' + + copy_common_archive_configs "$ARCHIVE_DEB" + +} +##################################### END ARCHIVE DEVNET PACKAGE ######################################## + +##################################### ARCHIVE BERKELEY PACKAGE ########################################### +build_archive_berkeley_deb () { + ARCHIVE_DEB=mina-archive-berkeley${DEB_SUFFIX} + + echo "------------------------------------------------------------" + echo "--- Building archive berkeley deb" + + + create_control_file "$ARCHIVE_DEB" "${ARCHIVE_DEPS}" 'Mina Archive Process + Compatible with Mina Daemon' + + copy_common_archive_configs "$ARCHIVE_DEB" } ##################################### END ARCHIVE PACKAGE ######################################## +##################################### ARCHIVE MAINNET PACKAGE ########################################### +build_archive_mainnet_deb () { + ARCHIVE_DEB=mina-archive-mainnet + + echo "------------------------------------------------------------" + echo "--- Building archive mainnet deb" + + create_control_file "$ARCHIVE_DEB" "${ARCHIVE_DEPS}" 'Mina Archive Process + Compatible with Mina Daemon' + + copy_common_archive_configs "$ARCHIVE_DEB" + +} +##################################### END ARCHIVE MAINNET PACKAGE ######################################## + ##################################### ZKAPP TEST TXN ############################################# build_zkapp_test_transaction_deb () { diff --git a/scripts/docker/helper.sh b/scripts/docker/helper.sh index 0e85d32f4c43..da6844f5f627 100644 --- a/scripts/docker/helper.sh +++ b/scripts/docker/helper.sh @@ -31,7 +31,7 @@ function export_suffixes () { # - lightnet # - hardfork-instrumented case "${DEB_PROFILE}" in - standard) + devnet|mainnet) case "${DEB_BUILD_FLAGS}" in *instrumented) export DOCKER_DEB_SUFFIX="--build-arg deb_suffix=instrumented" diff --git a/scripts/release-docker.sh b/scripts/release-docker.sh index 658b0f07f919..6fcff64f1512 100755 --- a/scripts/release-docker.sh +++ b/scripts/release-docker.sh @@ -62,38 +62,24 @@ case "${DEB_CODENAME##*=}" in esac IMAGE="--build-arg image=${IMAGE}" -# Determine suffix for mina name. Suffix is combined from profile and service name +# Determine suffix for mina name. Suffix is combined from network or build flags/profile +# but only for non standard builds. # Possible outcomes: -# - instrumented -# - hardfork -# - lightnet -# - hardfork-instrumented - case "${DEB_PROFILE}" in - standard) - case "${DEB_BUILD_FLAGS}" in - *instrumented) - DOCKER_DEB_SUFFIX="--build-arg deb_suffix=instrumented" - BUILD_FLAG_SUFFIX="-instrumented" - ;; - *) - ;; - esac - ;; - *) - case "${DEB_BUILD_FLAGS}" in - *instrumented) - DOCKER_DEB_SUFFIX="--build-arg deb_suffix=${DEB_PROFILE}-instrumented" - BUILD_FLAG_SUFFIX="-instrumented" - DEB_PROFILE_SUFFIX="-${DEB_PROFILE}" - ;; - *) - DOCKER_DEB_SUFFIX="--build-arg deb_suffix=${DEB_PROFILE}" - DEB_PROFILE_SUFFIX="-${DEB_PROFILE}" - ;; - esac - ;; - esac +# mina-berkeley-instrumented +# mina-archive-berkeley-instrumented +# mina-devnet +# mina-mainnet +if [[ ${NETWORK} == *"berkeley"* ]]; then + if [[ ${DEB_PROFILE} == *"lightnet"* ]]; then + DOCKER_DEB_SUFFIX="--build-arg deb_suffix=lightnet" + fi + if [[ ${DEB_BUILD_FLAGS} == *"instrumented"* ]]; then + DOCKER_DEB_SUFFIX="--build-arg deb_suffix=instrumented" + BUILD_FLAG_SUFFIX="-instrumented" + fi +fi + # Debug prints for visability # Substring removal to cut the --build-arg arguments on the = so that the output is exactly the input flags https://wiki.bash-hackers.org/syntax/pe#substring_removal echo "--service ${SERVICE} --version ${VERSION} --branch ${BRANCH##*=} --deb-version ${DEB_VERSION##*=} --deb-suffix ${DOCKER_DEB_SUFFIX##*=} --deb-release ${DEB_RELEASE##*=} --deb-codename ${DEB_CODENAME##*=}"