diff --git a/README.md b/README.md index c66605b8..25b44f8c 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ go build #### Example: build the docker image for gaia v6.0.0: ```bash -heighliner build -c gaia -v v6.0.0 +heighliner build --chain gaia --git-ref v6.0.0 ``` Docker image `heighliner/gaia:v6.0.0` will now be available in your local docker images @@ -40,7 +40,7 @@ Docker image `gaia:local` will be built and stored in your local docker images. ```bash # docker login ... -heighliner build -r ghcr.io/strangelove-ventures/heighliner -c gaia -v v6.0.0 +heighliner build -r ghcr.io/strangelove-ventures/heighliner -c gaia -g v6.0.0 ``` Docker image `ghcr.io/strangelove-ventures/heighliner/gaia:v6.0.0` will be built and pushed to ghcr.io @@ -77,7 +77,7 @@ Customize the platform(s) to be built with the `--platform` flag. #### Example: build x64 and arm64 docker images for gaia v7.0.1: ```bash -heighliner build -c gaia -v v7.0.1 +heighliner build -c gaia -g v7.0.1 ``` Docker images for `heighliner/gaia:v7.0.1` will now be available in your local docker. The manifest for the tag will contain both amd64 and arm64 images. @@ -85,7 +85,7 @@ Docker images for `heighliner/gaia:v7.0.1` will now be available in your local d #### Example: Use custom buildkit server, build x64 and arm64 docker images for gaia v7.0.1, and push: ```bash -heighliner build -b --buildkit-addr tcp://192.168.1.5:8125 -c gaia -v v7.0.1 -r ghcr.io/strangelove-ventures/heighliner +heighliner build -b --buildkit-addr tcp://192.168.1.5:8125 -c gaia -g v7.0.1 -r ghcr.io/strangelove-ventures/heighliner ``` Docker images for `heighliner/gaia:v7.0.1` will be built on the remote buildkit server and then pushed to the container repository. The manifest for the tag will contain both amd64 and arm64 images. diff --git a/addChain.md b/addChain.md index 4e2c6111..ee9fd430 100644 --- a/addChain.md +++ b/addChain.md @@ -15,13 +15,13 @@ Please keep chains in alphabetical order. `github-repo` -> The repo name of the location of the chain binary. -`language` -> Used in the Docker `FROM` argument to create a a docker base image. OPTIONS: "go, rust, nix, imported". Use "imported" if you are not able to build the chain binary from source and are importing a pre-made docker container. +`dockerfile` -> Which dockerfile strategy to use (folder names under dockerfile/). OPTIONS: `cosmos`, `cargo`, `imported`, or `none`. Use `imported` if you are importing an existing public docker image as a base for the heighliner image. Use `none` if you are not able to build the chain binary from source and need to download binaries into the image instead. `build-env` -> Environment variables to be created during the build. `pre-build` -> Any extra arguments needed to build the chain binary. -`build-target` -> The argument to call after `make` (language=golang), `cargo` (language=rust) or `nix` (language=nix). +`build-target` -> The build command specific to the chosen `dockerfile`. For `cosmos`, likely `make install`. For `cargo`, likely `build --release`. `binaries` -> The location of where the the build target places the binarie(s). Adding a ":" after the path allows for the ability to rename the binary. @@ -33,7 +33,7 @@ Please keep chains in alphabetical order. Please check the image builds successfully before submitting PR: -`./heighliner build -c -v +`./heighliner build -c -g ` Ensure binary runs in image: diff --git a/builder/builder.go b/builder/builder.go new file mode 100644 index 00000000..28234880 --- /dev/null +++ b/builder/builder.go @@ -0,0 +1,371 @@ +package builder + +import ( + "context" + "fmt" + "os" + "os/signal" + "path/filepath" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/strangelove-ventures/heighliner/docker" + "github.com/strangelove-ventures/heighliner/dockerfile" +) + +type HeighlinerBuilder struct { + buildConfig HeighlinerDockerBuildConfig + queue []HeighlinerQueuedChainBuilds + parallel int16 + local bool + + buildIndex int + buildIndexMu sync.Mutex + + errors []error + errorsLock sync.Mutex + + tmpDirsToRemove map[string]bool + tmpDirMapMu sync.Mutex +} + +func NewHeighlinerBuilder( + buildConfig HeighlinerDockerBuildConfig, + parallel int16, + local bool, +) *HeighlinerBuilder { + return &HeighlinerBuilder{ + buildConfig: buildConfig, + parallel: parallel, + local: local, + + tmpDirsToRemove: make(map[string]bool), + } +} + +func (h *HeighlinerBuilder) AddToQueue(chainBuilds ...HeighlinerQueuedChainBuilds) { + h.queue = append(h.queue, chainBuilds...) +} + +// imageTag determines which docker image tag to use based on inputs. +func imageTag(ref string, tag string, local bool) string { + if tag != "" { + return tag + } + + tag = deriveTagFromRef(ref) + + if local && tag == "" { + return "local" + } + + return tag +} + +// deriveTagFromRef returns a sanitized docker image tag from a git ref (branch/tag). +func deriveTagFromRef(version string) string { + return strings.ReplaceAll(version, "/", "-") +} + +// dockerfileEmbeddedOrLocal attempts to find Dockerfile within current working directory. +// Returns embedded Dockerfile if local file is not found or cannot be read. +func dockerfileEmbeddedOrLocal(dockerfile string, embedded []byte) []byte { + cwd, err := os.Getwd() + if err != nil { + fmt.Printf("Using embedded %s due to working directory not found\n", dockerfile) + return embedded + } + + absDockerfile := filepath.Join(cwd, "dockerfile", dockerfile) + if _, err := os.Stat(absDockerfile); err != nil { + fmt.Printf("Using embedded %s due to local dockerfile not found\n", dockerfile) + return embedded + } + + df, err := os.ReadFile(absDockerfile) + if err != nil { + fmt.Printf("Using embedded %s due to failure to read local dockerfile\n", dockerfile) + return embedded + } + + fmt.Printf("Using local %s\n", dockerfile) + return df +} + +// dockerfileAndTag returns the appropriate dockerfile as bytes and the docker image tag +// based on the input configuration. +func rawDockerfile( + dockerfileType DockerfileType, + useBuildKit bool, + local bool, +) []byte { + switch dockerfileType { + case DockerfileTypeImported: + return dockerfileEmbeddedOrLocal("imported/Dockerfile", dockerfile.Imported) + + case DockerfileTypeRust: + // DEPRECATED + fallthrough + case DockerfileTypeCargo: + if useBuildKit { + return dockerfileEmbeddedOrLocal("cargo/Dockerfile", dockerfile.Cargo) + } + return dockerfileEmbeddedOrLocal("cargo/native.Dockerfile", dockerfile.CargoNative) + + case DockerfileTypeGo: + // DEPRECATED + fallthrough + case DockerfileTypeCosmos: + if local { + // local builds always use embedded Dockerfile. + return dockerfile.CosmosLocal + } + if useBuildKit { + return dockerfileEmbeddedOrLocal("cosmos/Dockerfile", dockerfile.Cosmos) + } + return dockerfileEmbeddedOrLocal("cosmos/native.Dockerfile", dockerfile.CosmosNative) + + default: + return dockerfileEmbeddedOrLocal("none/Dockerfile", dockerfile.None) + } +} + +// buildChainNodeDockerImage builds the requested chain node docker image +// based on the input configuration. +func (h *HeighlinerBuilder) buildChainNodeDockerImage( + chainConfig *ChainNodeDockerBuildConfig, +) error { + buildCfg := h.buildConfig + dockerfile := chainConfig.Build.Dockerfile + + // DEPRECATION HANDLING + if chainConfig.Build.Language != "" { + fmt.Printf("'language' chain config property is deprecated, please use 'dockerfile' instead\n") + if dockerfile == "" { + dockerfile = chainConfig.Build.Language + } + } + + for _, rep := range deprecationReplacements { + if dockerfile == rep[0] { + fmt.Printf("'dockerfile' value of '%s' is deprecated, please use '%s' instead\n", rep[0], rep[1]) + } + } + // END DEPRECATION HANDLING + + df := rawDockerfile(dockerfile, buildCfg.UseBuildKit, h.local) + + tag := imageTag(chainConfig.Ref, chainConfig.Tag, h.local) + + cwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("error getting working directory: %w", err) + } + + dir, err := os.MkdirTemp(cwd, "heighliner") + if err != nil { + return fmt.Errorf("error making temporary directory for dockerfile: %w", err) + } + + // queue removal on ctrl+c + h.queueTmpDirRemoval(dir, true) + defer func() { + // this build is done, so don't need removal on ctrl+c anymore since we are removing now. + h.queueTmpDirRemoval(dir, false) + _ = os.RemoveAll(dir) + }() + + reldir, err := filepath.Rel(cwd, dir) + if err != nil { + return fmt.Errorf("error finding relative path for dockerfile working directory: %w", err) + } + + dfilepath := filepath.Join(reldir, "Dockerfile") + if err := os.WriteFile(dfilepath, df, 0644); err != nil { + return fmt.Errorf("error writing temporary dockerfile: %w", err) + } + + var imageName string + if buildCfg.ContainerRegistry == "" { + imageName = chainConfig.Build.Name + } else { + imageName = fmt.Sprintf("%s/%s", buildCfg.ContainerRegistry, chainConfig.Build.Name) + } + + imageTags := []string{fmt.Sprintf("%s:%s", imageName, tag)} + if chainConfig.Latest { + imageTags = append(imageTags, fmt.Sprintf("%s:latest", imageName)) + } + + fmt.Printf("Building image from ref: %s, tags: +%v\n", chainConfig.Ref, imageTags) + + buildEnv := "" + + buildTagsEnvVar := "" + for _, envVar := range chainConfig.Build.BuildEnv { + envVarSplit := strings.Split(envVar, "=") + if envVarSplit[0] == "BUILD_TAGS" { + buildTagsEnvVar = envVar + } else { + buildEnv += envVar + " " + } + } + + binaries := strings.Join(chainConfig.Build.Binaries, ",") + + libraries := strings.Join(chainConfig.Build.Libraries, " ") + + repoHost := chainConfig.Build.RepoHost + if repoHost == "" { + repoHost = "github.com" + } + + buildTimestamp := "" + if buildCfg.NoBuildCache { + buildTimestamp = strconv.FormatInt(time.Now().Unix(), 10) + } + + buildArgs := map[string]string{ + "VERSION": chainConfig.Ref, + "NAME": chainConfig.Build.Name, + "BASE_IMAGE": chainConfig.Build.BaseImage, + "REPO_HOST": repoHost, + "GITHUB_ORGANIZATION": chainConfig.Build.GithubOrganization, + "GITHUB_REPO": chainConfig.Build.GithubRepo, + "BUILD_TARGET": chainConfig.Build.BuildTarget, + "BINARIES": binaries, + "LIBRARIES": libraries, + "PRE_BUILD": chainConfig.Build.PreBuild, + "BUILD_ENV": buildEnv, + "BUILD_TAGS": buildTagsEnvVar, + "BUILD_DIR": chainConfig.Build.BuildDir, + "BUILD_TIMESTAMP": buildTimestamp, + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(time.Minute*180)) + defer cancel() + + push := buildCfg.ContainerRegistry != "" && !buildCfg.SkipPush + + if buildCfg.UseBuildKit { + buildKitOptions := docker.GetDefaultBuildKitOptions() + buildKitOptions.Address = buildCfg.BuildKitAddr + supportedPlatforms := chainConfig.Build.Platforms + + if len(supportedPlatforms) > 0 { + platforms := []string{} + requestedPlatforms := strings.Split(buildCfg.Platform, ",") + for _, supportedPlatform := range supportedPlatforms { + for _, requestedPlatform := range requestedPlatforms { + if supportedPlatform == requestedPlatform { + platforms = append(platforms, requestedPlatform) + } + } + } + if len(platforms) == 0 { + return fmt.Errorf("no requested platforms are supported for this chain: %s. requested: %s, supported: %s", chainConfig.Build.Name, buildCfg.Platform, strings.Join(supportedPlatforms, ",")) + } + buildKitOptions.Platform = strings.Join(platforms, ",") + } else { + buildKitOptions.Platform = buildCfg.Platform + } + buildKitOptions.NoCache = buildCfg.NoCache + if err := docker.BuildDockerImageWithBuildKit(ctx, reldir, imageTags, push, buildArgs, buildKitOptions); err != nil { + return err + } + } else { + if err := docker.BuildDockerImage(ctx, dfilepath, imageTags, push, buildArgs, buildCfg.NoCache); err != nil { + return err + } + } + return nil +} + +// returns queue items, starting with latest for each chain +func (h *HeighlinerBuilder) getNextQueueItem() *ChainNodeDockerBuildConfig { + h.buildIndexMu.Lock() + defer h.buildIndexMu.Unlock() + j := 0 + for i := 0; true; i++ { + foundForThisIndex := false + for _, queuedChainBuilds := range h.queue { + if i < len(queuedChainBuilds.ChainConfigs) { + if j == h.buildIndex { + h.buildIndex++ + return &queuedChainBuilds.ChainConfigs[i] + } + j++ + foundForThisIndex = true + } + } + if !foundForThisIndex { + // all done + return nil + } + } + return nil +} + +func (h *HeighlinerBuilder) buildNextImage(wg *sync.WaitGroup) { + chainConfig := h.getNextQueueItem() + if chainConfig == nil { + wg.Done() + return + } + + go func() { + if err := h.buildChainNodeDockerImage(chainConfig); err != nil { + h.errorsLock.Lock() + h.errors = append(h.errors, fmt.Errorf("error building docker image for %s from ref: %s - %v\n", chainConfig.Build.Name, chainConfig.Ref, err)) + h.errorsLock.Unlock() + } + h.buildNextImage(wg) + }() +} + +func (h *HeighlinerBuilder) queueTmpDirRemoval(tmpDir string, start bool) { + h.tmpDirMapMu.Lock() + defer h.tmpDirMapMu.Unlock() + if start { + h.tmpDirsToRemove[tmpDir] = true + } else { + delete(h.tmpDirsToRemove, tmpDir) + } +} + +// registerSigIntHandler will delete tmp dirs on ctrl+c +func (h *HeighlinerBuilder) registerSigIntHandler() { + c := make(chan os.Signal) + //nolint:govet + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go func() { + <-c + h.tmpDirMapMu.Lock() + defer h.tmpDirMapMu.Unlock() + for dir := range h.tmpDirsToRemove { + _ = os.RemoveAll(dir) + } + + os.Exit(1) + }() +} + +func (h *HeighlinerBuilder) BuildImages() { + h.registerSigIntHandler() + + wg := new(sync.WaitGroup) + for i := int16(0); i < h.parallel; i++ { + wg.Add(1) + h.buildNextImage(wg) + } + wg.Wait() + if len(h.errors) > 0 { + for _, err := range h.errors { + fmt.Println(err) + } + panic("Some images failed to build") + } +} diff --git a/builder/types.go b/builder/types.go new file mode 100644 index 00000000..74bb8fd3 --- /dev/null +++ b/builder/types.go @@ -0,0 +1,56 @@ +package builder + +type DockerfileType string + +const ( + DockerfileTypeCosmos DockerfileType = "cosmos" + DockerfileTypeCargo DockerfileType = "cargo" + DockerfileTypeImported DockerfileType = "imported" + + DockerfileTypeGo DockerfileType = "go" // DEPRECATED, use "cosmos" instead + DockerfileTypeRust DockerfileType = "rust" // DEPRECATED, use "cargo" instead +) + +// The first values for `dockerfile` are deprecated. Their recommended replacement is the second value. +var deprecationReplacements = [][2]DockerfileType{ + {DockerfileTypeGo, DockerfileTypeCosmos}, + {DockerfileTypeRust, DockerfileTypeCargo}, +} + +type ChainNodeConfig struct { + Name string `yaml:"name"` + RepoHost string `yaml:"repo-host"` + GithubOrganization string `yaml:"github-organization"` + GithubRepo string `yaml:"github-repo"` + Language DockerfileType `yaml:"language"` // DEPRECATED, use "dockerfile" instead + Dockerfile DockerfileType `yaml:"dockerfile"` + BuildTarget string `yaml:"build-target"` + BuildDir string `yaml:"build-dir"` + Binaries []string `yaml:"binaries"` + Libraries []string `yaml:"libraries"` + PreBuild string `yaml:"pre-build"` + Platforms []string `yaml:"platforms"` + BuildEnv []string `yaml:"build-env"` + BaseImage string `yaml:"base-image"` +} + +type ChainNodeDockerBuildConfig struct { + Build ChainNodeConfig + Ref string + Tag string + Latest bool +} + +type HeighlinerDockerBuildConfig struct { + ContainerRegistry string + SkipPush bool + UseBuildKit bool + BuildKitAddr string + Platform string + NoCache bool + NoBuildCache bool +} + +type HeighlinerQueuedChainBuilds struct { + ChainConfigs []ChainNodeDockerBuildConfig +} diff --git a/chains.yaml b/chains.yaml index 4b280437..96b96fae 100644 --- a/chains.yaml +++ b/chains.yaml @@ -1,13 +1,13 @@ # Agoric-sdk - name: agoric - language: imported + dockerfile: imported base-image: agoric/agoric-sdk # Akash - name: akash github-organization: ovrclk github-repo: akash - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/akash @@ -16,7 +16,7 @@ - name: assetmantle github-organization: assetmantle github-repo: node - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/mantleNode @@ -27,7 +27,7 @@ - name: axelar github-organization: axelarnetwork github-repo: axelar-core - language: go + dockerfile: cosmos binaries: - bin/axelard build-target: | @@ -58,7 +58,7 @@ - name: tofnd github-organization: axelarnetwork github-repo: tofnd - language: rust + dockerfile: cargo build-target: build --release pre-build: apt install -y libgmp3-dev:${TARGETARCH} @@ -69,7 +69,7 @@ - name: basilisk github-organization: galacticcouncil github-repo: Basilisk-node - language: rust + dockerfile: cargo build-target: build --release -Zbuild-std pre-build: | apt install -y zlib1g-dev:${TARGETARCH} @@ -83,7 +83,7 @@ - name: bitcanna github-organization: BitCannaGlobal github-repo: bcna - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/bcnad @@ -92,7 +92,7 @@ - name: bitsong github-organization: bitsongofficial github-repo: go-bitsong - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/bitsongd @@ -101,7 +101,7 @@ - name: bostrom github-organization: cybercongress github-repo: go-cyber - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/cyber @@ -112,7 +112,7 @@ - name: burnt github-organization: burnt-labs github-repo: burnt - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/burntd @@ -123,7 +123,7 @@ - name: carbon github-organization: Switcheo github-repo: carbon-bootstrap - language: rust + dockerfile: cargo pre-build: | apt update && apt install wget build-essential jq cmake sudo -y wget https://github.com/google/leveldb/archive/1.23.tar.gz && \ @@ -158,7 +158,7 @@ - name: cerberus github-organization: cerberus-zone github-repo: cerberus - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/cerberusd @@ -167,7 +167,7 @@ - name: cheqd github-organization: cheqd github-repo: cheqd-node - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/cheqd-noded @@ -176,7 +176,7 @@ - name: chihuahua github-organization: ChihuahuaChain github-repo: chihuahua - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/chihuahuad @@ -185,7 +185,7 @@ - name: comdex github-organization: comdex-official github-repo: comdex - language: go + dockerfile: cosmos build-target: make install build-env: - BUILD_TAGS=muslc @@ -196,7 +196,7 @@ - name: composable github-organization: ComposableFi github-repo: composable - language: rust + dockerfile: cargo pre-build: | apt install -y zlib1g-dev:${TARGETARCH} TOOLCHAIN=$(cat rust-toolchain.toml | grep channel | awk '{print $3}' | tr -d '"') @@ -223,7 +223,7 @@ - name: gaia github-organization: cosmos github-repo: gaia - language: go + dockerfile: cosmos build-target: make install build-env: - LEDGER_ENABLED=false @@ -235,7 +235,7 @@ - name: ics github-organization: cosmos github-repo: interchain-security - language: go + dockerfile: cosmos build-target: | export GOFLAGS='-buildmode=pie' export CGO_CPPFLAGS="-D_FORTIFY_SOURCE=2" @@ -250,7 +250,7 @@ - name: crescent github-organization: crescent-network github-repo: crescent - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/crescentd @@ -259,7 +259,7 @@ - name: cronos github-organization: crypto-org-chain github-repo: cronos - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/cronosd @@ -268,7 +268,7 @@ - name: cryptoorgchain github-organization: crypto-org-chain github-repo: chain-main - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/chain-maind @@ -277,7 +277,7 @@ - name: decentr github-organization: Decentr-net github-repo: decentr - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/decentrd @@ -286,7 +286,7 @@ - name: desmos github-organization: desmos-labs github-repo: desmos - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/desmos @@ -297,7 +297,7 @@ - name: dig github-organization: notional-labs github-repo: dig - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/digd @@ -308,7 +308,7 @@ - name: duality github-organization: duality-labs github-repo: duality - language: go + dockerfile: cosmos build-target: make install build-env: - LEDGER_ENABLED=false @@ -320,7 +320,7 @@ - name: emoney github-organization: e-money github-repo: em-ledger - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/emd @@ -329,7 +329,7 @@ - name: evmos github-organization: tharsis github-repo: evmos - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/evmosd @@ -338,7 +338,7 @@ - name: fetchhub github-organization: fetchai github-repo: fetchd - language: go + dockerfile: cosmos build-target: make install build-env: - BUILD_TAGS=muslc @@ -349,7 +349,7 @@ - name: firmachain github-organization: FirmaChain github-repo: firmachain - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/firmachaind @@ -358,7 +358,7 @@ - name: gravitybridge github-organization: Gravity-Bridge github-repo: Gravity-Bridge - language: go + dockerfile: cosmos build-target: make build build-dir: module binaries: @@ -368,7 +368,7 @@ - name: ibc-go-simd github-organization: cosmos github-repo: ibc-go - language: go + dockerfile: cosmos build-target: make build binaries: - build/simd @@ -379,7 +379,7 @@ - name: impacthub github-organization: ixofoundation github-repo: ixo-blockchain - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/ixod @@ -406,7 +406,7 @@ - name: icad github-organization: cosmos github-repo: interchain-accounts-demo - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/icad @@ -415,7 +415,7 @@ - name: icqd github-organization: quasar-finance github-repo: interchain-query-demo - language: go + dockerfile: cosmos build-target: go build -ldflags "$LDFLAGS" -o build/icq ./cmd/interchain-query-demod binaries: - build/icq @@ -424,7 +424,7 @@ - name: irisnet github-organization: irisnet github-repo: irishub - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/iris @@ -433,7 +433,7 @@ - name: juno github-organization: CosmosContracts github-repo: juno - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/junod @@ -445,7 +445,7 @@ - name: kichain github-organization: KiFoundation github-repo: ki-tools - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/kid @@ -456,7 +456,7 @@ - name: konstellation github-organization: knstl github-repo: konstellation - language: go + dockerfile: cosmos build-target: make install build-env: - BUILD_TAGS=muslc @@ -467,7 +467,7 @@ - name: kujira github-organization: Team-Kujira github-repo: core - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/kujirad @@ -479,7 +479,7 @@ - name: likecoin github-organization: likecoin github-repo: likecoin-chain - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/liked @@ -488,7 +488,7 @@ - name: lumnetwork github-organization: lum-network github-repo: chain - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/lumd @@ -497,7 +497,7 @@ - name: nomic github-organization: nomic-io github-repo: nomic - language: rust + dockerfile: cargo build-target: install --locked --path . -Zbuild-std pre-build: | TOOLCHAIN=$(cat rust-toolchain.toml | grep channel | awk '{print $3}' | tr -d '"') @@ -513,7 +513,7 @@ - name: neutron github-organization: neutron-org github-repo: neutron - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/neutrond @@ -524,7 +524,7 @@ - name: nibiru github-organization: NibiruChain github-repo: nibiru - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/nibid @@ -535,7 +535,7 @@ - name: omniflixhub github-organization: OmniFlix github-repo: omniflixhub - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/omniflixhubd @@ -544,7 +544,7 @@ - name: onomy github-organization: onomyprotocol github-repo: onomy - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/onomyd @@ -553,7 +553,7 @@ - name: osmosis github-organization: osmosis-labs github-repo: osmosis - language: go + dockerfile: cosmos build-target: make build binaries: - build/osmosisd @@ -564,7 +564,7 @@ - name: panacea github-organization: medibloc github-repo: panacea-core - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/panacead @@ -575,7 +575,7 @@ - name: penumbra github-organization: penumbra-zone github-repo: penumbra - language: rust + dockerfile: cargo build-target: build --release binaries: - /build/penumbra/target/${ARCH}-unknown-linux-gnu/release/pd @@ -585,7 +585,7 @@ - name: persistence github-organization: persistenceOne github-repo: persistenceCore - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/persistenceCore @@ -594,7 +594,7 @@ - name: polkadot github-organization: paritytech github-repo: polkadot - language: rust + dockerfile: cargo build-target: build --release pre-build: | ./scripts/init.sh @@ -605,7 +605,7 @@ - name: provenance github-organization: provenance-io github-repo: provenance - language: go + dockerfile: cosmos build-target: make install pre-build: | apk add --no-cache g++ @@ -626,7 +626,7 @@ - name: quicksilver github-organization: ingenuity-build github-repo: quicksilver - language: go + dockerfile: cosmos build-target: make build binaries: - build/quicksilverd @@ -635,7 +635,7 @@ - name: regen github-organization: regen-network github-repo: regen-ledger - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/regen @@ -644,14 +644,14 @@ - name: rizon github-organization: rizon-world github-repo: rizon - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/rizond # Secret Network - name: secretnetwork - language: imported + dockerfile: imported base-image: ghcr.io/scrtlabs/secret-network-node platforms: - linux/amd64 @@ -660,7 +660,7 @@ - name: sentinel github-organization: sentinel-official github-repo: hub - language: go + dockerfile: cosmos # Sentinel Makefile does not consume LDFLAGS or BUILD_TAGS env vars. build-target: | BUILD_TAGS=netgo,muslc @@ -673,7 +673,7 @@ - name: shentu github-organization: ShentuChain github-repo: shentu - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/certik @@ -682,7 +682,7 @@ - name: sifchain github-organization: Sifchain github-repo: sifnode - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/sifnoded @@ -691,7 +691,7 @@ - name: sim github-organization: cosmos github-repo: cosmos-sdk - language: go + dockerfile: cosmos build-target: make build binaries: - build/simd @@ -700,7 +700,7 @@ - name: sommelier github-organization: peggyjv github-repo: sommelier - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/sommelier @@ -709,7 +709,7 @@ - name: stargaze github-organization: public-awesome github-repo: stargaze - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/starsd @@ -720,7 +720,7 @@ - name: starname github-organization: iov-one github-repo: starnamed - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/starnamed @@ -731,7 +731,7 @@ - name: stride github-organization: Stride-Labs github-repo: stride - language: go + dockerfile: cosmos build-target: make build binaries: - build/strided @@ -740,7 +740,7 @@ - name: tendermint github-organization: tendermint github-repo: tendermint - language: go + dockerfile: cosmos build-target: make build binaries: - /go/src/github.com/tendermint/tendermint/build/tendermint @@ -749,7 +749,7 @@ - name: terra github-organization: terra-money github-repo: core - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/terrad @@ -761,7 +761,7 @@ repo-host: gitlab.com github-organization: thorchain github-repo: thornode - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/thornode @@ -773,7 +773,7 @@ - name: umee github-organization: umee-network github-repo: umee - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/umeed @@ -782,7 +782,7 @@ - name: vidulum github-organization: vidulum github-repo: mainnet - language: go + dockerfile: cosmos build-target: make install binaries: - /go/bin/vidulumd @@ -791,7 +791,7 @@ - name: wasm github-organization: CosmWasm github-repo: wasmd - language: go + dockerfile: cosmos build-target: make build binaries: - build/wasmd diff --git a/cmd/build.go b/cmd/build.go index 49d01728..bda38a23 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -1,310 +1,35 @@ package cmd import ( - "context" - "encoding/json" "fmt" - "io" - "log" - "net/http" "os" - "os/signal" "path/filepath" - "strconv" - "strings" - "sync" - "syscall" - "time" "github.com/spf13/cobra" + "github.com/strangelove-ventures/heighliner/builder" "github.com/strangelove-ventures/heighliner/docker" - "github.com/strangelove-ventures/heighliner/dockerfile" "gopkg.in/yaml.v2" ) -type ChainNodeConfig struct { - Name string `yaml:"name"` - RepoHost string `yaml:"repo-host"` - GithubOrganization string `yaml:"github-organization"` - GithubRepo string `yaml:"github-repo"` - Language string `yaml:"language"` - BuildTarget string `yaml:"build-target"` - BuildDir string `yaml:"build-dir"` - Binaries []string `yaml:"binaries"` - Libraries []string `yaml:"libraries"` - PreBuild string `yaml:"pre-build"` - Platforms []string `yaml:"platforms"` - BuildEnv []string `yaml:"build-env"` - BaseImage string `yaml:"base-image"` -} - -type GithubRelease struct { - TagName string `json:"tag_name"` -} - -type ChainNodeDockerBuildConfig struct { - Build ChainNodeConfig - Version string - Latest bool -} - -type HeighlinerDockerBuildConfig struct { - ContainerRegistry string - SkipPush bool - UseBuildKit bool - BuildKitAddr string - Platform string - NoCache bool - NoBuildCache bool -} - -type HeighlinerQueuedChainBuilds struct { - ChainConfigs []ChainNodeDockerBuildConfig -} - -// tagFromVersion returns a sanitized docker image tag from a version string. -func tagFromVersion(version string) string { - return strings.ReplaceAll(version, "/", "-") -} - -// getDockerfile attempts to find Dockerfile within current working directory. -// Returns embedded Dockerfile if local file is not found or cannot be read. -func getDockerfile(dockerfile string, embedded []byte) []byte { - cwd, err := os.Getwd() - if err != nil { - fmt.Printf("Using embedded %s due to working directory not found\n", dockerfile) - return embedded - } - - absDockerfile := filepath.Join(cwd, "dockerfile", dockerfile) - if _, err := os.Stat(absDockerfile); err != nil { - fmt.Printf("Using embedded %s due to local dockerfile not found\n", dockerfile) - return embedded - } - - df, err := os.ReadFile(absDockerfile) - if err != nil { - fmt.Printf("Using embedded %s due to failure to read local dockerfile\n", dockerfile) - return embedded - } - - fmt.Printf("Using local %s", dockerfile) - return df -} - -// dockerfileAndTag returns the appropriate dockerfile as bytes and the docker image tag -// based on the input configuration. -func dockerfileAndTag( - buildConfig *HeighlinerDockerBuildConfig, - chainConfig *ChainNodeDockerBuildConfig, - local bool, -) ([]byte, string) { - switch chainConfig.Build.Language { - case "imported": - return getDockerfile("imported/Dockerfile", dockerfile.Imported), tagFromVersion(chainConfig.Version) - case "rust": - if buildConfig.UseBuildKit { - return getDockerfile("rust/Dockerfile", dockerfile.Rust), tagFromVersion(chainConfig.Version) - } - return getDockerfile("rust/native.Dockerfile", dockerfile.RustNative), tagFromVersion(chainConfig.Version) - case "go": - if local { - // local builds always use embedded Dockerfile. - if chainConfig.Version == "" { - return dockerfile.SDKLocal, "local" - } - return dockerfile.SDKLocal, tagFromVersion(chainConfig.Version) - } - if buildConfig.UseBuildKit { - return getDockerfile("sdk/Dockerfile", dockerfile.SDK), tagFromVersion(chainConfig.Version) - } - return getDockerfile("sdk/native.Dockerfile", dockerfile.SDKNative), tagFromVersion(chainConfig.Version) - default: - return getDockerfile("none/Dockerfile", dockerfile.None), tagFromVersion(chainConfig.Version) - } -} - -// buildChainNodeDockerImage builds the requested chain node docker image -// based on the input configuration. -func buildChainNodeDockerImage( - buildConfig *HeighlinerDockerBuildConfig, - chainConfig *ChainNodeDockerBuildConfig, - local bool, - queueTmpDirRemoval func(tmpDir string, start bool), -) error { - df, imageTag := dockerfileAndTag(buildConfig, chainConfig, local) - - cwd, err := os.Getwd() - if err != nil { - return fmt.Errorf("error getting working directory: %w", err) - } - - dir, err := os.MkdirTemp(cwd, "heighliner") - if err != nil { - return fmt.Errorf("error making temporary directory for dockerfile: %w", err) - } - - // queue removal on ctrl+c - queueTmpDirRemoval(dir, true) - defer func() { - // this build is done, so don't need removal on ctrl+c anymore since we are removing now. - queueTmpDirRemoval(dir, false) - _ = os.RemoveAll(dir) - }() - - reldir, err := filepath.Rel(cwd, dir) - if err != nil { - return fmt.Errorf("error finding relative path for dockerfile working directory: %w", err) - } - - dfilepath := filepath.Join(reldir, "Dockerfile") - if err := os.WriteFile(dfilepath, df, 0644); err != nil { - return fmt.Errorf("error writing temporary dockerfile: %w", err) - } - - var imageName string - if buildConfig.ContainerRegistry == "" { - imageName = chainConfig.Build.Name - } else { - imageName = fmt.Sprintf("%s/%s", buildConfig.ContainerRegistry, chainConfig.Build.Name) - } - - imageTags := []string{fmt.Sprintf("%s:%s", imageName, imageTag)} - if chainConfig.Latest { - imageTags = append(imageTags, fmt.Sprintf("%s:latest", imageName)) - } - - fmt.Printf("Image Tags: +%v\n", imageTags) - - buildEnv := "" - - buildTagsEnvVar := "" - for _, envVar := range chainConfig.Build.BuildEnv { - envVarSplit := strings.Split(envVar, "=") - if envVarSplit[0] == "BUILD_TAGS" { - buildTagsEnvVar = envVar - } else { - buildEnv += envVar + " " - } - } - - binaries := strings.Join(chainConfig.Build.Binaries, ",") - - libraries := strings.Join(chainConfig.Build.Libraries, " ") - - repoHost := chainConfig.Build.RepoHost - if repoHost == "" { - repoHost = "github.com" - } - - buildTimestamp := "" - if buildConfig.NoBuildCache { - buildTimestamp = strconv.FormatInt(time.Now().Unix(), 10) - } - - buildArgs := map[string]string{ - "VERSION": chainConfig.Version, - "NAME": chainConfig.Build.Name, - "BASE_IMAGE": chainConfig.Build.BaseImage, - "REPO_HOST": repoHost, - "GITHUB_ORGANIZATION": chainConfig.Build.GithubOrganization, - "GITHUB_REPO": chainConfig.Build.GithubRepo, - "BUILD_TARGET": chainConfig.Build.BuildTarget, - "BINARIES": binaries, - "LIBRARIES": libraries, - "PRE_BUILD": chainConfig.Build.PreBuild, - "BUILD_ENV": buildEnv, - "BUILD_TAGS": buildTagsEnvVar, - "BUILD_DIR": chainConfig.Build.BuildDir, - "BUILD_TIMESTAMP": buildTimestamp, - } - - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(time.Minute*180)) - defer cancel() - - push := buildConfig.ContainerRegistry != "" && !buildConfig.SkipPush - - if buildConfig.UseBuildKit { - buildKitOptions := docker.GetDefaultBuildKitOptions() - buildKitOptions.Address = buildConfig.BuildKitAddr - supportedPlatforms := chainConfig.Build.Platforms - - if len(supportedPlatforms) > 0 { - platforms := []string{} - requestedPlatforms := strings.Split(buildConfig.Platform, ",") - for _, supportedPlatform := range supportedPlatforms { - for _, requestedPlatform := range requestedPlatforms { - if supportedPlatform == requestedPlatform { - platforms = append(platforms, requestedPlatform) - } - } - } - if len(platforms) == 0 { - return fmt.Errorf("no requested platforms are supported for this chain: %s. requested: %s, supported: %s", chainConfig.Build.Name, buildConfig.Platform, strings.Join(supportedPlatforms, ",")) - } - buildKitOptions.Platform = strings.Join(platforms, ",") - } else { - buildKitOptions.Platform = buildConfig.Platform - } - buildKitOptions.NoCache = buildConfig.NoCache - if err := docker.BuildDockerImageWithBuildKit(ctx, reldir, imageTags, push, buildArgs, buildKitOptions); err != nil { - return err - } - } else { - if err := docker.BuildDockerImage(ctx, dfilepath, imageTags, push, buildArgs, buildConfig.NoCache); err != nil { - return err - } - } - return nil -} - -func queueMostRecentReleasesForChain( - chainQueuedBuilds *HeighlinerQueuedChainBuilds, - chainNodeConfig ChainNodeConfig, - number int16, -) error { - if chainNodeConfig.GithubOrganization == "" || chainNodeConfig.GithubRepo == "" { - return fmt.Errorf("github organization: %s and/or repo: %s not provided for chain: %s\n", chainNodeConfig.GithubOrganization, chainNodeConfig.GithubRepo, chainNodeConfig.Name) - } - client := http.Client{Timeout: 5 * time.Second} - - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://api.github.com/repos/%s/%s/releases?per_page=%d&page=1", - chainNodeConfig.GithubOrganization, chainNodeConfig.GithubRepo, number), http.NoBody) - if err != nil { - return fmt.Errorf("error building github releases request: %v", err) - } - - basicAuthUser := os.Getenv("GITHUB_USER") - basicAuthPassword := os.Getenv("GITHUB_PASSWORD") - - req.SetBasicAuth(basicAuthUser, basicAuthPassword) - - res, err := client.Do(req) - if err != nil { - return fmt.Errorf("error performing github releases request: %v", err) - } - - defer res.Body.Close() - - body, err := io.ReadAll(res.Body) - if err != nil { - return fmt.Errorf("error reading body from github releases request: %v", err) - } - - releases := []GithubRelease{} - err = json.Unmarshal(body, &releases) - if err != nil { - return fmt.Errorf("error parsing github releases response: %s, error: %v", body, err) - } - for i, release := range releases { - chainQueuedBuilds.ChainConfigs = append(chainQueuedBuilds.ChainConfigs, ChainNodeDockerBuildConfig{ - Build: chainNodeConfig, - Version: release.TagName, - Latest: i == 0, - }) - } - return nil -} +const ( + flagFile = "file" + flagRegistry = "registry" + flagChain = "chain" + flagOrg = "org" + flagGitRef = "git-ref" + flagTag = "tag" + flagVersion = "version" // DEPRECATED + flagNumber = "number" + flagParallel = "parallel" + flagSkip = "skip" + flagLatest = "latest" + flagLocal = "local" + flagUseBuildkit = "use-buildkit" + flagBuildkitAddr = "buildkit-addr" + flagPlatform = "platform" + flagNoCache = "no-cache" + flagNoBuildCache = "no-build-cache" +) func loadChainsYaml(configFile string) error { if _, err := os.Stat(configFile); err != nil { @@ -314,7 +39,7 @@ func loadChainsYaml(configFile string) error { if err != nil { return fmt.Errorf("error reading file: %s: %w", configFile, err) } - var newChains []ChainNodeConfig + var newChains []builder.ChainNodeConfig err = yaml.Unmarshal(bz, &newChains) if err != nil { return fmt.Errorf("error unmarshalling yaml from file: %s: %w", configFile, err) @@ -332,7 +57,7 @@ it will be built and pushed`, Run: func(cmd *cobra.Command, args []string) { cmdFlags := cmd.Flags() - configFile, _ := cmdFlags.GetString("file") + configFile, _ := cmdFlags.GetString(flagFile) if configFile == "" { // try to load a local chains.yaml, but do not panic for any error, will fall back to embedded chains. cwd, err := os.Getwd() @@ -351,24 +76,25 @@ it will be built and pushed`, } } - containerRegistry, _ := cmdFlags.GetString("registry") - chain, _ := cmdFlags.GetString("chain") - version, _ := cmdFlags.GetString("version") - org, _ := cmdFlags.GetString("org") - number, _ := cmdFlags.GetInt16("number") - skip, _ := cmdFlags.GetBool("skip") - - useBuildKit, _ := cmdFlags.GetBool("use-buildkit") - buildKitAddr, _ := cmdFlags.GetString("buildkit-addr") - platform, _ := cmdFlags.GetString("platform") - noCache, _ := cmdFlags.GetBool("no-cache") - noBuildCache, _ := cmdFlags.GetBool("no-build-cache") - latest, _ := cmdFlags.GetBool("latest") - local, _ := cmdFlags.GetBool("local") - parallel, _ := cmdFlags.GetInt16("parallel") - - buildQueue := []*HeighlinerQueuedChainBuilds{} - buildConfig := HeighlinerDockerBuildConfig{ + containerRegistry, _ := cmdFlags.GetString(flagRegistry) + chain, _ := cmdFlags.GetString(flagChain) + version, _ := cmdFlags.GetString(flagVersion) + ref, _ := cmdFlags.GetString(flagGitRef) + tag, _ := cmdFlags.GetString(flagTag) + org, _ := cmdFlags.GetString(flagOrg) + number, _ := cmdFlags.GetInt16(flagNumber) + skip, _ := cmdFlags.GetBool(flagSkip) + + useBuildKit, _ := cmdFlags.GetBool(flagUseBuildkit) + buildKitAddr, _ := cmdFlags.GetString(flagBuildkitAddr) + platform, _ := cmdFlags.GetString(flagPlatform) + noCache, _ := cmdFlags.GetBool(flagNoCache) + noBuildCache, _ := cmdFlags.GetBool(flagNoBuildCache) + latest, _ := cmdFlags.GetBool(flagLatest) + local, _ := cmdFlags.GetBool(flagLocal) + parallel, _ := cmdFlags.GetInt16(flagParallel) + + buildConfig := builder.HeighlinerDockerBuildConfig{ ContainerRegistry: containerRegistry, SkipPush: skip, UseBuildKit: useBuildKit, @@ -378,157 +104,43 @@ it will be built and pushed`, NoBuildCache: noBuildCache, } - for _, chainNodeConfig := range chains { - // If chain is provided, only build images for that chain - // Chain must be declared in chains.yaml - if chain != "" && chainNodeConfig.Name != chain { - continue + // DEPRECATION HANDLING + if version != "" { + if ref == "" { + ref = version } - if org != "" { - chainNodeConfig.GithubOrganization = org - } - chainQueuedBuilds := HeighlinerQueuedChainBuilds{ChainConfigs: []ChainNodeDockerBuildConfig{}} - if version != "" || local { - chainConfig := ChainNodeDockerBuildConfig{ - Build: chainNodeConfig, - Version: version, - Latest: latest, - } - chainQueuedBuilds.ChainConfigs = append(chainQueuedBuilds.ChainConfigs, chainConfig) - buildQueue = append(buildQueue, &chainQueuedBuilds) - buildImages(&buildConfig, buildQueue, parallel, local) - return - } - // If specific version not provided, build images for the last n releases from the chain - err := queueMostRecentReleasesForChain(&chainQueuedBuilds, chainNodeConfig, number) - if err != nil { - log.Printf("Error queueing docker image builds for chain %s: %v", chainNodeConfig.Name, err) - continue - } - if len(chainQueuedBuilds.ChainConfigs) > 0 { - buildQueue = append(buildQueue, &chainQueuedBuilds) - } - } - buildImages(&buildConfig, buildQueue, parallel, false) - }, -} - -// returns queue items, starting with latest for each chain -func getQueueItem(queue []*HeighlinerQueuedChainBuilds, index int) *ChainNodeDockerBuildConfig { - j := 0 - for i := 0; true; i++ { - foundForThisIndex := false - for _, queuedChainBuilds := range queue { - if i < len(queuedChainBuilds.ChainConfigs) { - if j == index { - return &queuedChainBuilds.ChainConfigs[i] - } - j++ - foundForThisIndex = true - } - } - if !foundForThisIndex { - // all done - return nil - } - } - return nil -} - -func buildNextImage( - buildConfig *HeighlinerDockerBuildConfig, - queue []*HeighlinerQueuedChainBuilds, - buildIndex *int, - buildIndexLock *sync.Mutex, - wg *sync.WaitGroup, - errors *[]error, - errorsLock *sync.Mutex, - local bool, - queueTmpDirRemoval func(tmpDir string, start bool), -) { - buildIndexLock.Lock() - defer buildIndexLock.Unlock() - chainConfig := getQueueItem(queue, *buildIndex) - *buildIndex++ - if chainConfig == nil { - wg.Done() - return - } - go func() { - log.Printf("Building docker image: %s:%s\n", chainConfig.Build.Name, chainConfig.Version) - if err := buildChainNodeDockerImage(buildConfig, chainConfig, local, queueTmpDirRemoval); err != nil { - errorsLock.Lock() - *errors = append(*errors, fmt.Errorf("error building docker image for %s:%s - %v\n", chainConfig.Build.Name, chainConfig.Version, err)) - errorsLock.Unlock() - } - buildNextImage(buildConfig, queue, buildIndex, buildIndexLock, wg, errors, errorsLock, local, queueTmpDirRemoval) - }() -} - -func buildImages(buildConfig *HeighlinerDockerBuildConfig, queue []*HeighlinerQueuedChainBuilds, parallel int16, local bool) { - buildIndex := 0 - buildIndexLock := sync.Mutex{} - errors := []error{} - errorsLock := sync.Mutex{} - - tmpDirsToRemove := make(map[string]bool) - var tmpDirMapMu sync.Mutex - queueTmpDirRemoval := func(tmpDir string, start bool) { - tmpDirMapMu.Lock() - defer tmpDirMapMu.Unlock() - if start { - tmpDirsToRemove[tmpDir] = true - } else { - delete(tmpDirsToRemove, tmpDir) - } - } - - // Delete tmp dirs on ctrl+c - c := make(chan os.Signal) - //nolint:govet - signal.Notify(c, os.Interrupt, syscall.SIGTERM) - go func() { - <-c - tmpDirMapMu.Lock() - defer tmpDirMapMu.Unlock() - for dir := range tmpDirsToRemove { - _ = os.RemoveAll(dir) + fmt.Printf( + `Warning: --version/-v flag is deprecated. Please update to use --git-ref/-g instead. +An optional flag --tag/-t is now available to override the resulting docker image tag if desirable to differ from the derived tag +`) } + // END DEPRECATION HANDLING - os.Exit(1) - }() - - wg := sync.WaitGroup{} - for i := int16(0); i < parallel; i++ { - wg.Add(1) - buildNextImage(buildConfig, queue, &buildIndex, &buildIndexLock, &wg, &errors, &errorsLock, local, queueTmpDirRemoval) - } - wg.Wait() - if len(errors) > 0 { - for _, err := range errors { - log.Println(err) - } - panic("Some images failed to build") - } + queueAndBuild(buildConfig, chain, org, ref, tag, latest, local, number, parallel) + }, } func init() { rootCmd.AddCommand(buildCmd) - buildCmd.PersistentFlags().StringP("file", "f", "", "chains.yaml config file path") - buildCmd.PersistentFlags().StringP("registry", "r", "", "Docker Container Registry for pushing images") - buildCmd.PersistentFlags().StringP("chain", "c", "", "Cosmos chain to build from chains.yaml") - buildCmd.PersistentFlags().StringP("org", "o", "", "Github organization override for building from a fork") - buildCmd.PersistentFlags().StringP("version", "v", "", "Github tag to build") - buildCmd.PersistentFlags().Int16P("number", "n", 5, "Number of releases to build per chain") - buildCmd.PersistentFlags().Int16("parallel", 1, "Number of docker builds to run simultaneously") - buildCmd.PersistentFlags().BoolP("skip", "s", false, "Skip pushing images to registry") - buildCmd.PersistentFlags().BoolP("latest", "l", false, "Also push latest tag (for single version build only)") - buildCmd.PersistentFlags().Bool("local", false, "Use local directory (not git repository)") - - buildCmd.PersistentFlags().BoolP("use-buildkit", "b", false, "Use buildkit to build multi-arch images") - buildCmd.PersistentFlags().String("buildkit-addr", docker.BuildKitSock, "Address of the buildkit socket, can be unix, tcp, ssl") - buildCmd.PersistentFlags().StringP("platform", "p", docker.DefaultPlatforms, "Platforms to build") - buildCmd.PersistentFlags().Bool("no-cache", false, "Don't use docker cache for building") - buildCmd.PersistentFlags().Bool("no-build-cache", false, "Invalidate caches for clone and build.") + buildCmd.PersistentFlags().StringP(flagFile, "f", "", "chains.yaml config file path") + buildCmd.PersistentFlags().StringP(flagRegistry, "r", "", "Docker Container Registry for pushing images") + buildCmd.PersistentFlags().StringP(flagChain, "c", "", "Cosmos chain to build from chains.yaml") + buildCmd.PersistentFlags().StringP(flagOrg, "o", "", "Github organization override for building from a fork") + buildCmd.PersistentFlags().StringP(flagGitRef, "g", "", "Github short ref to build (branch, tag)") + buildCmd.PersistentFlags().StringP(flagTag, "t", "", "Resulting docker image tag. If not provided, will derive from ref.") + buildCmd.PersistentFlags().Int16P(flagNumber, "n", 5, "Number of releases to build per chain") + buildCmd.PersistentFlags().Int16(flagParallel, 1, "Number of docker builds to run simultaneously") + buildCmd.PersistentFlags().BoolP(flagSkip, "s", false, "Skip pushing images to registry") + buildCmd.PersistentFlags().BoolP(flagLatest, "l", false, "Also push latest tag (for single version build only)") + buildCmd.PersistentFlags().Bool(flagLocal, false, "Use local directory (not git repository)") + + buildCmd.PersistentFlags().BoolP(flagUseBuildkit, "b", false, "Use buildkit to build multi-arch images") + buildCmd.PersistentFlags().String(flagBuildkitAddr, docker.BuildKitSock, "Address of the buildkit socket, can be unix, tcp, ssl") + buildCmd.PersistentFlags().StringP(flagPlatform, "p", docker.DefaultPlatforms, "Platforms to build (only applies to buildkit builds with -b)") + buildCmd.PersistentFlags().Bool(flagNoCache, false, "Don't use docker cache for building") + buildCmd.PersistentFlags().Bool(flagNoBuildCache, false, "Invalidate caches for clone and build.") + + // DEPRECATED + buildCmd.PersistentFlags().StringP(flagVersion, "v", "", "DEPRECATED, use --git-ref/-g instead") } diff --git a/cmd/queue.go b/cmd/queue.go new file mode 100644 index 00000000..278cac4d --- /dev/null +++ b/cmd/queue.go @@ -0,0 +1,114 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/strangelove-ventures/heighliner/builder" +) + +type GithubRelease struct { + TagName string `json:"tag_name"` +} + +func mostRecentReleasesForChain( + chainNodeConfig builder.ChainNodeConfig, + number int16, +) (builder.HeighlinerQueuedChainBuilds, error) { + if chainNodeConfig.GithubOrganization == "" || chainNodeConfig.GithubRepo == "" { + return builder.HeighlinerQueuedChainBuilds{}, fmt.Errorf("github organization: %s and/or repo: %s not provided for chain: %s\n", chainNodeConfig.GithubOrganization, chainNodeConfig.GithubRepo, chainNodeConfig.Name) + } + client := http.Client{Timeout: 5 * time.Second} + + if chainNodeConfig.RepoHost != "" && chainNodeConfig.RepoHost != "github.com" { + return builder.HeighlinerQueuedChainBuilds{}, nil + } + + fmt.Printf("Fetching most recent releases for github.com/%s/%s\n", chainNodeConfig.GithubOrganization, chainNodeConfig.GithubRepo) + + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://api.github.com/repos/%s/%s/releases?per_page=%d&page=1", + chainNodeConfig.GithubOrganization, chainNodeConfig.GithubRepo, number), http.NoBody) + if err != nil { + return builder.HeighlinerQueuedChainBuilds{}, fmt.Errorf("error building github releases request: %v", err) + } + + res, err := client.Do(req) + if err != nil { + return builder.HeighlinerQueuedChainBuilds{}, fmt.Errorf("error performing github releases request: %v", err) + } + + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + if err != nil { + return builder.HeighlinerQueuedChainBuilds{}, fmt.Errorf("error reading body from github releases request: %v", err) + } + + releases := []GithubRelease{} + err = json.Unmarshal(body, &releases) + if err != nil { + return builder.HeighlinerQueuedChainBuilds{}, fmt.Errorf("error parsing github releases response: %s, error: %v", body, err) + } + chainQueuedBuilds := builder.HeighlinerQueuedChainBuilds{} + for i, release := range releases { + fmt.Printf("Adding release tag to build queue: %s\n", release.TagName) + chainQueuedBuilds.ChainConfigs = append(chainQueuedBuilds.ChainConfigs, builder.ChainNodeDockerBuildConfig{ + Build: chainNodeConfig, + Ref: release.TagName, + Latest: i == 0, + }) + } + + return chainQueuedBuilds, nil +} + +func queueAndBuild( + buildConfig builder.HeighlinerDockerBuildConfig, + chain string, + org string, + ref string, + tag string, + latest bool, + local bool, + number int16, + parallel int16, +) { + heighlinerBuilder := builder.NewHeighlinerBuilder(buildConfig, parallel, local) + + for _, chainNodeConfig := range chains { + // If chain is provided, only build images for that chain + // Chain must be declared in chains.yaml + if chain != "" && chainNodeConfig.Name != chain { + continue + } + if org != "" { + chainNodeConfig.GithubOrganization = org + } + chainQueuedBuilds := builder.HeighlinerQueuedChainBuilds{ChainConfigs: []builder.ChainNodeDockerBuildConfig{}} + if ref != "" || local { + chainConfig := builder.ChainNodeDockerBuildConfig{ + Build: chainNodeConfig, + Ref: ref, + Tag: tag, + Latest: latest, + } + chainQueuedBuilds.ChainConfigs = append(chainQueuedBuilds.ChainConfigs, chainConfig) + heighlinerBuilder.AddToQueue(chainQueuedBuilds) + heighlinerBuilder.BuildImages() + return + } + // If specific version not provided, build images for the last n releases from the chain + chainBuilds, err := mostRecentReleasesForChain(chainNodeConfig, number) + if err != nil { + fmt.Printf("Error queueing docker image builds for chain %s: %v", chainNodeConfig.Name, err) + continue + } + if len(chainBuilds.ChainConfigs) > 0 { + heighlinerBuilder.AddToQueue(chainBuilds) + } + } + heighlinerBuilder.BuildImages() +} diff --git a/cmd/root.go b/cmd/root.go index 9c960655..c526db14 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,10 +1,11 @@ package cmd import ( - "log" + "fmt" "os" "github.com/spf13/cobra" + "github.com/strangelove-ventures/heighliner/builder" "gopkg.in/yaml.v2" ) @@ -17,12 +18,12 @@ This tool can generate docker images for all different release versions of the configured Cosmos blockchains in chains.yaml`, } -var chains []ChainNodeConfig +var chains []builder.ChainNodeConfig func Execute(chainsYaml []byte) { err := yaml.Unmarshal(chainsYaml, &chains) if err != nil { - log.Fatalf("Error parsing chains.yaml: %v", err) + panic(fmt.Errorf("error parsing chains.yaml: %v", err)) } err = rootCmd.Execute() diff --git a/docker/docker.go b/docker/docker.go index 9a40474e..80467529 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "io" - "log" "strings" "github.com/docker/docker/api/types" @@ -54,7 +53,7 @@ func BuildDockerImage(ctx context.Context, dockerfile string, tags []string, pus tar, err := archive.TarWithOptions("./", &archive.TarOptions{}) if err != nil { - log.Fatalf("Error archiving project for docker: %v", err) + panic(fmt.Errorf("error archiving project for docker: %v", err)) } res, err := dockerClient.ImageBuild(ctx, tar, opts) diff --git a/dockerfile/rust/Dockerfile b/dockerfile/cargo/Dockerfile similarity index 100% rename from dockerfile/rust/Dockerfile rename to dockerfile/cargo/Dockerfile diff --git a/dockerfile/rust/native.Dockerfile b/dockerfile/cargo/native.Dockerfile similarity index 100% rename from dockerfile/rust/native.Dockerfile rename to dockerfile/cargo/native.Dockerfile diff --git a/dockerfile/sdk/Dockerfile b/dockerfile/cosmos/Dockerfile similarity index 100% rename from dockerfile/sdk/Dockerfile rename to dockerfile/cosmos/Dockerfile diff --git a/dockerfile/sdk/local.Dockerfile b/dockerfile/cosmos/local.Dockerfile similarity index 100% rename from dockerfile/sdk/local.Dockerfile rename to dockerfile/cosmos/local.Dockerfile diff --git a/dockerfile/sdk/native.Dockerfile b/dockerfile/cosmos/native.Dockerfile similarity index 100% rename from dockerfile/sdk/native.Dockerfile rename to dockerfile/cosmos/native.Dockerfile diff --git a/dockerfile/dockerfiles.go b/dockerfile/dockerfiles.go index 59eea40b..5415cea6 100644 --- a/dockerfile/dockerfiles.go +++ b/dockerfile/dockerfiles.go @@ -2,14 +2,14 @@ package dockerfile import _ "embed" -//go:embed sdk/Dockerfile -var SDK []byte +//go:embed cosmos/Dockerfile +var Cosmos []byte -//go:embed sdk/native.Dockerfile -var SDKNative []byte +//go:embed cosmos/native.Dockerfile +var CosmosNative []byte -//go:embed sdk/local.Dockerfile -var SDKLocal []byte +//go:embed cosmos/local.Dockerfile +var CosmosLocal []byte //go:embed imported/Dockerfile var Imported []byte @@ -17,8 +17,8 @@ var Imported []byte //go:embed none/Dockerfile var None []byte -//go:embed rust/Dockerfile -var Rust []byte +//go:embed cargo/Dockerfile +var Cargo []byte -//go:embed rust/native.Dockerfile -var RustNative []byte +//go:embed cargo/native.Dockerfile +var CargoNative []byte