Skip to content

use mainnet profile in mainnet artifacts build #17166

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 27 commits into
base: compatible
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
29389f8
use mainnet profile in mainnet artifacts build
dkijania May 8, 2025
d76c69f
be more specific which profile is used
dkijania May 8, 2025
7e34dfa
build archive for 3 networks
dkijania May 8, 2025
304a4be
add network suffix to archive debian
dkijania May 9, 2025
b5934aa
add suffixes only for berkeley
dkijania May 9, 2025
bed8f05
some more comments
dkijania May 9, 2025
8facd5f
fix suffixes
dkijania May 10, 2025
a66de02
Update builder-helpers.sh
dkijania May 10, 2025
72fec80
Update helper.sh
dkijania May 10, 2025
c1a93f9
Update Dockerfile-mina-rosetta
dkijania May 10, 2025
64698bd
Update MinaArtifact.dhall
dkijania May 10, 2025
7f4b571
Update MinaArtifact.dhall
dkijania May 10, 2025
2655b1c
Update Dockerfile-mina-rosetta
dkijania May 10, 2025
6999cf3
fix suffix for archive / daemon dockers
dkijania May 10, 2025
29d2dd5
last fix for dependencies
dkijania May 10, 2025
25ae7bd
fix instrumented jobs deps
dkijania May 10, 2025
56b4467
fix deps
dkijania May 11, 2025
ee87fb1
Update Base.dhall
dkijania May 12, 2025
4d20733
Update PatchArchiveTest.dhall
dkijania May 12, 2025
07d2bbf
Update ReplayerTest.dhall
dkijania May 12, 2025
554ad68
fiz Base.dhall depends usage
dkijania May 12, 2025
3730e22
fix network var
dkijania May 12, 2025
18a1846
fix network in rosetta int tests
dkijania May 12, 2025
0c0bb35
name -> dockerName
dkijania May 12, 2025
8d54d5b
fix single node tests deps and mina-test-suite related deps
dkijania May 13, 2025
9eaf8f7
fix imports
dkijania May 13, 2025
74f3149
fix rosetta dep
dkijania May 13, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions buildkite/src/Command/Bench/Base.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ let RunInToolchain = ../../Command/RunInToolchain.dhall

let Network = ../../Constants/Network.dhall

let BuildFlags = ../../Constants/BuildFlags.dhall

let Profiles = ../../Constants/Profiles.dhall

let Command = ../../Command/Base.dhall
Expand Down Expand Up @@ -49,10 +51,13 @@ let Spec =
{ mode = PipelineMode.Type.PullRequest
, size = Size.Perf
, dependsOn =
DebianVersions.dependsOn
DebianVersions.dependsOnStep
(None Text)
DebianVersions.DebVersion.Bullseye
Network.Type.Berkeley
Profiles.Type.Standard
Profiles.Type.Devnet
BuildFlags.Type.Instrumented
"build"
, additionalDirtyWhen = [] : List SelectFiles.Type
, yellowThreshold = 0.1
, redThreshold = 0.2
Expand Down
2 changes: 1 addition & 1 deletion buildkite/src/Command/Bench/LedgerApply.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ let dependsOn =
(None Text)
DebianVersions.DebVersion.Bullseye
Network.Type.Berkeley
Profiles.Type.Standard
Profiles.Type.Devnet
BuildFlags.Type.Instrumented
"build"

Expand Down
7 changes: 2 additions & 5 deletions buildkite/src/Command/DockerImage.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ let ReleaseSpec =
, deb_codename = DebianVersions.DebVersion.Bullseye
, deb_release = "\\\${MINA_DEB_RELEASE}"
, deb_version = "\\\${MINA_DEB_VERSION}"
, deb_profile = Profiles.Type.Standard
, deb_profile = Profiles.Type.Devnet
, build_flags = BuildFlags.Type.None
, deb_repo = DebianRepo.Type.Local
, docker_publish = DockerPublish.Type.Essential
Expand All @@ -69,10 +69,7 @@ let ReleaseSpec =

let stepKey =
\(spec : ReleaseSpec.Type)
-> "${Artifacts.lowerName
spec.service}${Profiles.toLabelSegment
spec.deb_profile}${BuildFlags.toLabelSegment
spec.build_flags}${spec.step_key_suffix}"
-> "${Artifacts.lowerName spec.service}${spec.step_key_suffix}"

let stepLabel =
\(spec : ReleaseSpec.Type)
Expand Down
2 changes: 1 addition & 1 deletion buildkite/src/Command/HardforkPackageGeneration.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ let Spec =
, genesis_timestamp = Some "2024-04-07T11:45:00Z"
, config_json_gz_url =
"https://storage.googleapis.com/o1labs-gitops-infrastructure/devnet/devnet-state-dump-3NK4eDgbkCjKj9fFUXVkrJXsfpfXzJySoAvrFJVCropPW7LLF14F-676026c4d4d2c18a76b357d6422a06f932c3ef4667a8fd88717f68b53fd6b2d7.json.gz"
, profile = Profiles.Type.Standard
, profile = Profiles.Type.Devnet
, suffix = "hardfork"
}
}
Expand Down
10 changes: 2 additions & 8 deletions buildkite/src/Command/MinaArtifact.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ let MinaBuildSpec =
{ prefix = "MinaArtifact"
, artifacts = Artifacts.AllButTests
, debVersion = DebianVersions.DebVersion.Bullseye
, profile = Profiles.Type.Standard
, profile = Profiles.Type.Devnet
, buildFlags = BuildFlags.Type.None
, network = Network.Type.Berkeley
, toolchainSelectMode = Toolchain.SelectionMode.ByDebian
Expand Down Expand Up @@ -83,12 +83,6 @@ let nameSuffix
spec.profile}${BuildFlags.toSuffixUppercase
spec.buildFlags}"

let keySuffix
: MinaBuildSpec.Type -> Text
= \(spec : MinaBuildSpec.Type)
-> "${Profiles.toLabelSegment spec.profile}${BuildFlags.toLabelSegment
spec.buildFlags}"

let build_artifacts
: MinaBuildSpec.Type -> Command.Type
= \(spec : MinaBuildSpec.Type)
Expand Down Expand Up @@ -207,6 +201,7 @@ let docker_step
, service = Artifacts.Type.Archive
, deb_codename = spec.debVersion
, deb_profile = spec.profile
, network = Network.lowerName spec.network
, build_flags = spec.buildFlags
, docker_publish = docker_publish
, deb_repo = DebianRepo.Type.Local
Expand Down Expand Up @@ -322,5 +317,4 @@ in { pipeline = pipeline
, publishToDebian = publish_to_debian_repo
, MinaBuildSpec = MinaBuildSpec
, labelSuffix = labelSuffix
, keySuffix = keySuffix
}
5 changes: 3 additions & 2 deletions buildkite/src/Command/PatchArchiveTest.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ let Size = ./Size.dhall

let Network = ../Constants/Network.dhall

let BuildFlags = ../Constants/BuildFlags.dhall

let RunWithPostgres = ./RunWithPostgres.dhall

let key = "patch-archive-test"
Expand All @@ -20,8 +22,7 @@ in { step =
, "NETWORK_DATA_FOLDER=/etc/mina/test/archive/sample_db"
]
"./src/test/archive/sample_db/archive_db.sql"
Artifacts.Type.FunctionalTestSuite
(None Network.Type)
"gcr.io/o1labs-192920/${Artifacts.dockerName Artifacts.Type.FunctionalTestSuite}:\\\$MINA_DOCKER_TAG-${Network.lowerName Network.Type.Berkeley}-${BuildFlags.lowerName BuildFlags.Type.Instrumented}"
"./scripts/patch-archive-test.sh && buildkite/scripts/upload-partial-coverage-data.sh ${key}"
]
, label = "Archive: Patch Archive test"
Expand Down
4 changes: 2 additions & 2 deletions buildkite/src/Command/Promotion/PromoteDebian.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ let PromoteDebianSpec =
, to_channel = DebianChannel.Type.Compatible
, source_repo = DebianRepo.Type.Unstable
, target_repo = DebianRepo.Type.Nightly
, profile = Profiles.Type.Standard
, profile = Profiles.Type.Devnet
, remove_profile_from_name = False
, step_key = "promote-debian-package"
, if = None B/If
Expand All @@ -68,7 +68,7 @@ let promoteDebianStep =

then "--new-name ${Package.debianName
spec.package
Profiles.Type.Standard
Profiles.Type.Devnet
spec.network}"

else ""
Expand Down
2 changes: 1 addition & 1 deletion buildkite/src/Command/Promotion/PromoteDocker.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ let PromoteDockerSpec =
, version = ""
, new_tags = [] : List Text
, step_key = "promote-docker"
, profile = Profiles.Type.Standard
, profile = Profiles.Type.Devnet
, network = Network.Type.Devnet
, codename = DebianVersions.DebVersion.Bullseye
, if = None B/If
Expand Down
2 changes: 1 addition & 1 deletion buildkite/src/Command/Promotion/PromotePackages.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ let PromotePackagesSpec =
, source_debian_repo = DebianRepo.Type.Unstable
, target_debian_repo = DebianRepo.Type.Nightly
, architecture = "amd64"
, profile = Profiles.Type.Standard
, profile = Profiles.Type.Devnet
, network = Network.Type.Mainnet
, codenames = [] : List DebianVersions.DebVersion
, from_channel = DebianChannel.Type.Unstable
Expand Down
2 changes: 1 addition & 1 deletion buildkite/src/Command/Promotion/VerifyDebian.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ let promoteDebianVerificationStep =

then "${Package.debianName
spec.package
Profiles.Type.Standard
Profiles.Type.Devnet
spec.network}"

else Package.debianName spec.package spec.profile spec.network
Expand Down
2 changes: 1 addition & 1 deletion buildkite/src/Command/Promotion/VerifyPackages.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ let VerifyPackagesSpec =
, dockers = [] : List Artifact.Type
, new_debian_version = "\\\\\$MINA_DEB_VERSION"
, debian_repo = DebianRepo.Type.Local
, profile = Profiles.Type.Standard
, profile = Profiles.Type.Devnet
, network = Network.Type.Mainnet
, codenames = [] : List DebianVersions.DebVersion
, channel = DebianChannel.Type.Compatible
Expand Down
5 changes: 3 additions & 2 deletions buildkite/src/Command/ReplayerTest.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ let Command = ./Base.dhall

let Size = ./Size.dhall

let BuildFlags = ../Constants/BuildFlags.dhall

let RunWithPostgres = ./RunWithPostgres.dhall

let Network = ../Constants/Network.dhall
Expand All @@ -18,8 +20,7 @@ in { step =
[ RunWithPostgres.runInDockerWithPostgresConn
([] : List Text)
"./src/test/archive/sample_db/archive_db.sql"
Artifacts.Type.FunctionalTestSuite
(None Network.Type)
"gcr.io/o1labs-192920/${Artifacts.dockerName Artifacts.Type.FunctionalTestSuite}:\\\$MINA_DOCKER_TAG-${Network.lowerName Network.Type.Berkeley}-${BuildFlags.lowerName BuildFlags.Type.Instrumented}"
"./buildkite/scripts/replayer-test.sh && buildkite/scripts/upload-partial-coverage-data.sh ${key}"
]
, label = "Archive: Replayer test"
Expand Down
2 changes: 1 addition & 1 deletion buildkite/src/Command/Rosetta/Connectivity.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ let command
Dockers.dependsOn
spec.dockerType
spec.network
Profiles.Type.Standard
(Profiles.fromNetwork spec.network)
Artifacts.Type.Rosetta
}

Expand Down
22 changes: 3 additions & 19 deletions buildkite/src/Command/RunWithPostgres.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,12 @@ let Network = ../Constants/Network.dhall
let runInDockerWithPostgresConn
: List Text
-> Text
-> Artifacts.Type
-> Optional Network.Type
-> Text
-> Text
-> Cmd.Type
= \(environment : List Text)
-> \(initScript : Text)
-> \(docker : Artifacts.Type)
-> \(network : Optional Network.Type)
-> \(docker : Text)
-> \(innerScript : Text)
-> let port = "5432"

Expand Down Expand Up @@ -60,27 +58,13 @@ let runInDockerWithPostgresConn
: Text
= "\\\$BUILDKITE_BUILD_CHECKOUT_PATH"

let minaDockerTag
: Text
= "\\\$MINA_DOCKER_TAG"

let maybeNetwork =
Optional/map
Network.Type
Text
(\(network : Network.Type) -> "-${Network.lowerName network}")
network

let networkOrDefault = Optional/default Text "" maybeNetwork

in Cmd.chain
[ "( docker stop ${postgresDockerName} && docker rm ${postgresDockerName} ) || true"
, "source buildkite/scripts/export-git-env-vars.sh"
, "docker run --network host --volume ${outerDir}:/workdir --workdir /workdir --name ${postgresDockerName} -d -e POSTGRES_USER=${user} -e POSTGRES_PASSWORD=${password} -e POSTGRES_PASSWORD=${password} -e POSTGRES_DB=${dbName} ${dockerVersion}"
, "sleep 5"
, "docker exec ${postgresDockerName} psql ${pg_conn} -f /workdir/${initScript}"
, "docker run --network host --volume ${outerDir}:/workdir --workdir /workdir --entrypoint bash ${envVars} gcr.io/o1labs-192920/${Artifacts.dockerName
docker}:${minaDockerTag}${networkOrDefault} ${innerScript}"
, "docker run --network host --volume ${outerDir}:/workdir --workdir /workdir --entrypoint bash ${envVars} ${docker} ${innerScript}"
]

in { runInDockerWithPostgresConn = runInDockerWithPostgresConn }
4 changes: 2 additions & 2 deletions buildkite/src/Constants/Artifacts.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ let toDebianName =
-> merge
{ Daemon = "daemon_${Network.lowerName network}"
, LogProc = "logproc"
, Archive = "archive"
, Archive = "archive_${Network.lowerName network}"
, TestExecutive = "test_executive"
, BatchTxn = "batch_txn"
, Rosetta = "rosetta_${Network.lowerName network}"
Expand All @@ -108,7 +108,7 @@ let toDebianNames =
( \(a : Artifact)
-> merge
{ Daemon = [ toDebianName a network ]
, Archive = [ "archive" ]
, Archive = [ toDebianName a network ]
, LogProc = [ "logproc" ]
, TestExecutive = [ "test_executive" ]
, BatchTxn = [ "batch_txn" ]
Expand Down
21 changes: 16 additions & 5 deletions buildkite/src/Constants/DockerVersions.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ let Artifacts = ./Artifacts.dhall

let Network = ./Network.dhall

let BuildFlags = ./BuildFlags.dhall

let Docker
: Type
= < Bookworm | Bullseye | Jammy | Focal >
Expand Down Expand Up @@ -33,9 +35,12 @@ let dependsOnStep =
-> \(prefix : Text)
-> \(network : Network.Type)
-> \(profile : Profiles.Type)
-> \(buildFlag : BuildFlags.Type)
-> \(binary : Artifacts.Type)
-> let network = "${Network.capitalName network}"

let buildFlag = "${BuildFlags.toSuffixUppercase buildFlag}"

let profileSuffix = "${Profiles.toSuffixUppercase profile}"

let suffix = "docker-image"
Expand All @@ -46,30 +51,30 @@ let dependsOnStep =
{ Bookworm =
[ { name =
"${prefix}${capitalName
docker}${network}${profileSuffix}"
docker}${network}${profileSuffix}${buildFlag}"
, key = key
}
]
, Bullseye =
[ { name =
"${prefix}${capitalName
docker}${network}${profileSuffix}"
docker}${network}${profileSuffix}${buildFlag}"
, key = key
}
]
, Jammy =
[ { name =
"${prefix}${capitalName
docker}${network}${capitalName
docker}${profileSuffix}"
docker}${profileSuffix}${buildFlag}"
, key = key
}
]
, Focal =
[ { name =
"${prefix}${capitalName
docker}${network}${capitalName
docker}${profileSuffix}"
docker}${profileSuffix}${buildFlag}"
, key = key
}
]
Expand All @@ -81,7 +86,13 @@ let dependsOn =
-> \(network : Network.Type)
-> \(profile : Profiles.Type)
-> \(binary : Artifacts.Type)
-> dependsOnStep docker "MinaArtifact" network profile binary
-> dependsOnStep
docker
"MinaArtifact"
network
profile
BuildFlags.Type.None
binary

in { Type = Docker
, capitalName = capitalName
Expand Down
Loading