From 52a44deca34ff672e1328d511c202c5e5deb82bd Mon Sep 17 00:00:00 2001 From: Pasquale Congiusti Date: Wed, 11 Oct 2023 12:57:14 +0200 Subject: [PATCH] feat(trait): disable incremental image option Closes #3011 --- .../camel.apache.org_integrationkits.yaml | 4 ++ ...camel.apache.org_integrationplatforms.yaml | 8 ++++ .../bases/camel.apache.org_integrations.yaml | 4 ++ .../camel.apache.org_kameletbindings.yaml | 4 ++ config/crd/bases/camel.apache.org_pipes.yaml | 4 ++ .../pages/architecture/incremental-image.adoc | 4 +- .../ROOT/partials/apis/camel-k-crds.adoc | 7 +++ docs/modules/traits/pages/builder.adoc | 4 ++ .../incremental_build_test.go | 48 +++++++++++++++++++ helm/camel-k/crds/crd-integration-kit.yaml | 4 ++ .../crds/crd-integration-platform.yaml | 8 ++++ helm/camel-k/crds/crd-integration.yaml | 4 ++ helm/camel-k/crds/crd-kamelet-binding.yaml | 4 ++ helm/camel-k/crds/crd-pipe.yaml | 4 ++ pkg/apis/camel/v1/trait/builder.go | 2 + .../camel/v1/trait/zz_generated.deepcopy.go | 5 ++ pkg/trait/builder.go | 6 ++- pkg/trait/quarkus.go | 17 ++++++- pkg/trait/quarkus_test.go | 4 +- resources/traits.yaml | 4 ++ 20 files changed, 145 insertions(+), 4 deletions(-) diff --git a/config/crd/bases/camel.apache.org_integrationkits.yaml b/config/crd/bases/camel.apache.org_integrationkits.yaml index 5c282381be..fade604f6a 100644 --- a/config/crd/bases/camel.apache.org_integrationkits.yaml +++ b/config/crd/bases/camel.apache.org_integrationkits.yaml @@ -203,6 +203,10 @@ spec: description: Can be used to enable or disable a trait. All traits share this common property. type: boolean + incrementalImageBuild: + description: Use the incremental image build option, to reuse + existing containers (default `true`) + type: boolean limitCPU: description: 'When using `pod` strategy, the maximum amount of CPU required by the pod builder. Deprecated: use TasksRequestCPU diff --git a/config/crd/bases/camel.apache.org_integrationplatforms.yaml b/config/crd/bases/camel.apache.org_integrationplatforms.yaml index 50f8854b8c..2ba99f660f 100644 --- a/config/crd/bases/camel.apache.org_integrationplatforms.yaml +++ b/config/crd/bases/camel.apache.org_integrationplatforms.yaml @@ -506,6 +506,10 @@ spec: description: Can be used to enable or disable a trait. All traits share this common property. type: boolean + incrementalImageBuild: + description: Use the incremental image build option, to reuse + existing containers (default `true`) + type: boolean limitCPU: description: 'When using `pod` strategy, the maximum amount of CPU required by the pod builder. Deprecated: use TasksRequestCPU @@ -2328,6 +2332,10 @@ spec: description: Can be used to enable or disable a trait. All traits share this common property. type: boolean + incrementalImageBuild: + description: Use the incremental image build option, to reuse + existing containers (default `true`) + type: boolean limitCPU: description: 'When using `pod` strategy, the maximum amount of CPU required by the pod builder. Deprecated: use TasksRequestCPU diff --git a/config/crd/bases/camel.apache.org_integrations.yaml b/config/crd/bases/camel.apache.org_integrations.yaml index 5589263a27..fd022c6438 100644 --- a/config/crd/bases/camel.apache.org_integrations.yaml +++ b/config/crd/bases/camel.apache.org_integrations.yaml @@ -6423,6 +6423,10 @@ spec: description: Can be used to enable or disable a trait. All traits share this common property. type: boolean + incrementalImageBuild: + description: Use the incremental image build option, to reuse + existing containers (default `true`) + type: boolean limitCPU: description: 'When using `pod` strategy, the maximum amount of CPU required by the pod builder. Deprecated: use TasksRequestCPU diff --git a/config/crd/bases/camel.apache.org_kameletbindings.yaml b/config/crd/bases/camel.apache.org_kameletbindings.yaml index 5f5f20eead..8ff36efee1 100644 --- a/config/crd/bases/camel.apache.org_kameletbindings.yaml +++ b/config/crd/bases/camel.apache.org_kameletbindings.yaml @@ -6705,6 +6705,10 @@ spec: description: Can be used to enable or disable a trait. All traits share this common property. type: boolean + incrementalImageBuild: + description: Use the incremental image build option, to + reuse existing containers (default `true`) + type: boolean limitCPU: description: 'When using `pod` strategy, the maximum amount of CPU required by the pod builder. Deprecated: use diff --git a/config/crd/bases/camel.apache.org_pipes.yaml b/config/crd/bases/camel.apache.org_pipes.yaml index 793ba658f2..8980595486 100644 --- a/config/crd/bases/camel.apache.org_pipes.yaml +++ b/config/crd/bases/camel.apache.org_pipes.yaml @@ -6703,6 +6703,10 @@ spec: description: Can be used to enable or disable a trait. All traits share this common property. type: boolean + incrementalImageBuild: + description: Use the incremental image build option, to + reuse existing containers (default `true`) + type: boolean limitCPU: description: 'When using `pod` strategy, the maximum amount of CPU required by the pod builder. Deprecated: use diff --git a/docs/modules/ROOT/pages/architecture/incremental-image.adoc b/docs/modules/ROOT/pages/architecture/incremental-image.adoc index 6e84545468..50fcdcbd6d 100644 --- a/docs/modules/ROOT/pages/architecture/incremental-image.adoc +++ b/docs/modules/ROOT/pages/architecture/incremental-image.adoc @@ -6,4 +6,6 @@ In order to reduce the time to build and execute an application we adopt various If you're familiar with Camel K architecture, you know that an Integration uses an IntegrationKit, which is, a reusable resource containing all the dependencies and capabilities required to run a certain "class" of Camel applications. The IntegrationKit is bound to a container image stored in a registry. -When the IntegrationKit is created, it uses as a base image a JDK based container and add certain applications layers on top of it. However, instead of using always the same root base image, we check if an IntegrationKit with a subset of dependencies already exists, sparing quite some time on the generation of the new container image. \ No newline at end of file +When the IntegrationKit is created, it uses as a base image a JDK based container and add certain applications layers on top of it. However, instead of using always the same root base image, we check if an IntegrationKit with a subset of dependencies already exists, sparing quite some time on the generation of the new container image. + +NOTE: you may disable the incremental image feature and always build and package your Camel application from scratch using builder trait option `-t builder.incremental-image-build=false`. \ No newline at end of file diff --git a/docs/modules/ROOT/partials/apis/camel-k-crds.adoc b/docs/modules/ROOT/partials/apis/camel-k-crds.adoc index 9b6408d5b1..b2aaca8138 100644 --- a/docs/modules/ROOT/partials/apis/camel-k-crds.adoc +++ b/docs/modules/ROOT/partials/apis/camel-k-crds.adoc @@ -5789,6 +5789,13 @@ string The strategy to use, either `pod` or `routine` (default routine) +|`incrementalImageBuild` + +bool +| + + +Use the incremental image build option, to reuse existing containers (default `true`) + |`orderStrategy` + string | diff --git a/docs/modules/traits/pages/builder.adoc b/docs/modules/traits/pages/builder.adoc index e990024ab3..584ee4ad91 100755 --- a/docs/modules/traits/pages/builder.adoc +++ b/docs/modules/traits/pages/builder.adoc @@ -40,6 +40,10 @@ The following configuration options are available: | string | The strategy to use, either `pod` or `routine` (default routine) +| builder.incremental-image-build +| bool +| Use the incremental image build option, to reuse existing containers (default `true`) + | builder.order-strategy | string | The build order strategy to use, either `dependencies`, `fifo` or `sequential` (default sequential) diff --git a/e2e/commonwithcustominstall/incremental_build_test.go b/e2e/commonwithcustominstall/incremental_build_test.go index fe68dc9d7f..d6e94f9044 100644 --- a/e2e/commonwithcustominstall/incremental_build_test.go +++ b/e2e/commonwithcustominstall/incremental_build_test.go @@ -137,3 +137,51 @@ func TestRunIncrementalBuildPod(t *testing.T) { Expect(Kamel("delete", "--all", "-n", ns).Execute()).To(Succeed()) }) } + +func TestRunIncrementalBuildOff(t *testing.T) { + WithNewTestNamespace(t, func(ns string) { + operatorID := "camel-k-standard-build" + Expect(KamelInstallWithID(operatorID, ns).Execute()).To(Succeed()) + + name := "java" + Expect(KamelRunWithID(operatorID, ns, "files/Java.java", + "--name", name, + ).Execute()).To(Succeed()) + Eventually(IntegrationPodPhase(ns, name), TestTimeoutLong).Should(Equal(corev1.PodRunning)) + Eventually(IntegrationConditionStatus(ns, name, v1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(corev1.ConditionTrue)) + Eventually(IntegrationLogs(ns, name), TestTimeoutShort).Should(ContainSubstring("Magicstring!")) + integrationKitName := IntegrationKit(ns, name)() + Eventually(Kit(ns, integrationKitName)().Status.BaseImage).Should(Equal(defaults.BaseImage())) + + t.Run("Don't reuse previous kit", func(t *testing.T) { + nameClone := "java-clone" + Expect(KamelRunWithID(operatorID, ns, "files/Java.java", + "--name", nameClone, + "-t", "builder.incremental-image-build=false", + ).Execute()).To(Succeed()) + Eventually(IntegrationPodPhase(ns, nameClone), TestTimeoutLong).Should(Equal(corev1.PodRunning)) + Eventually(IntegrationConditionStatus(ns, nameClone, v1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(corev1.ConditionTrue)) + Eventually(IntegrationLogs(ns, nameClone), TestTimeoutShort).Should(ContainSubstring("Magicstring!")) + integrationCloneKitName := IntegrationKit(ns, nameClone)() + Eventually(Kit(ns, integrationCloneKitName)().Status.BaseImage).Should(Equal(defaults.BaseImage())) + }) + + t.Run("Don't create incremental kit", func(t *testing.T) { + // Another integration that should be built on top of the previous IntegrationKit + // just add a new random dependency + nameIncremental := "java-incremental" + Expect(KamelRunWithID(operatorID, ns, "files/Java.java", + "--name", nameIncremental, + "-d", "camel:zipfile", + "-t", "builder.incremental-image-build=false", + ).Execute()).To(Succeed()) + Eventually(IntegrationPodPhase(ns, nameIncremental), TestTimeoutLong).Should(Equal(corev1.PodRunning)) + Eventually(IntegrationConditionStatus(ns, nameIncremental, v1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(corev1.ConditionTrue)) + Eventually(IntegrationLogs(ns, nameIncremental), TestTimeoutShort).Should(ContainSubstring("Magicstring!")) + integrationIncrementalKitName := IntegrationKit(ns, nameIncremental)() + Eventually(Kit(ns, integrationIncrementalKitName)().Status.BaseImage).Should(Equal(defaults.BaseImage())) + }) + + Expect(Kamel("delete", "--all", "-n", ns).Execute()).To(Succeed()) + }) +} diff --git a/helm/camel-k/crds/crd-integration-kit.yaml b/helm/camel-k/crds/crd-integration-kit.yaml index 5c282381be..fade604f6a 100644 --- a/helm/camel-k/crds/crd-integration-kit.yaml +++ b/helm/camel-k/crds/crd-integration-kit.yaml @@ -203,6 +203,10 @@ spec: description: Can be used to enable or disable a trait. All traits share this common property. type: boolean + incrementalImageBuild: + description: Use the incremental image build option, to reuse + existing containers (default `true`) + type: boolean limitCPU: description: 'When using `pod` strategy, the maximum amount of CPU required by the pod builder. Deprecated: use TasksRequestCPU diff --git a/helm/camel-k/crds/crd-integration-platform.yaml b/helm/camel-k/crds/crd-integration-platform.yaml index 50f8854b8c..2ba99f660f 100644 --- a/helm/camel-k/crds/crd-integration-platform.yaml +++ b/helm/camel-k/crds/crd-integration-platform.yaml @@ -506,6 +506,10 @@ spec: description: Can be used to enable or disable a trait. All traits share this common property. type: boolean + incrementalImageBuild: + description: Use the incremental image build option, to reuse + existing containers (default `true`) + type: boolean limitCPU: description: 'When using `pod` strategy, the maximum amount of CPU required by the pod builder. Deprecated: use TasksRequestCPU @@ -2328,6 +2332,10 @@ spec: description: Can be used to enable or disable a trait. All traits share this common property. type: boolean + incrementalImageBuild: + description: Use the incremental image build option, to reuse + existing containers (default `true`) + type: boolean limitCPU: description: 'When using `pod` strategy, the maximum amount of CPU required by the pod builder. Deprecated: use TasksRequestCPU diff --git a/helm/camel-k/crds/crd-integration.yaml b/helm/camel-k/crds/crd-integration.yaml index 5589263a27..fd022c6438 100644 --- a/helm/camel-k/crds/crd-integration.yaml +++ b/helm/camel-k/crds/crd-integration.yaml @@ -6423,6 +6423,10 @@ spec: description: Can be used to enable or disable a trait. All traits share this common property. type: boolean + incrementalImageBuild: + description: Use the incremental image build option, to reuse + existing containers (default `true`) + type: boolean limitCPU: description: 'When using `pod` strategy, the maximum amount of CPU required by the pod builder. Deprecated: use TasksRequestCPU diff --git a/helm/camel-k/crds/crd-kamelet-binding.yaml b/helm/camel-k/crds/crd-kamelet-binding.yaml index 5f5f20eead..8ff36efee1 100644 --- a/helm/camel-k/crds/crd-kamelet-binding.yaml +++ b/helm/camel-k/crds/crd-kamelet-binding.yaml @@ -6705,6 +6705,10 @@ spec: description: Can be used to enable or disable a trait. All traits share this common property. type: boolean + incrementalImageBuild: + description: Use the incremental image build option, to + reuse existing containers (default `true`) + type: boolean limitCPU: description: 'When using `pod` strategy, the maximum amount of CPU required by the pod builder. Deprecated: use diff --git a/helm/camel-k/crds/crd-pipe.yaml b/helm/camel-k/crds/crd-pipe.yaml index 793ba658f2..8980595486 100644 --- a/helm/camel-k/crds/crd-pipe.yaml +++ b/helm/camel-k/crds/crd-pipe.yaml @@ -6703,6 +6703,10 @@ spec: description: Can be used to enable or disable a trait. All traits share this common property. type: boolean + incrementalImageBuild: + description: Use the incremental image build option, to + reuse existing containers (default `true`) + type: boolean limitCPU: description: 'When using `pod` strategy, the maximum amount of CPU required by the pod builder. Deprecated: use diff --git a/pkg/apis/camel/v1/trait/builder.go b/pkg/apis/camel/v1/trait/builder.go index 38d4b07294..edc031dfe7 100644 --- a/pkg/apis/camel/v1/trait/builder.go +++ b/pkg/apis/camel/v1/trait/builder.go @@ -29,6 +29,8 @@ type BuilderTrait struct { Properties []string `property:"properties" json:"properties,omitempty"` // The strategy to use, either `pod` or `routine` (default routine) Strategy string `property:"strategy" json:"strategy,omitempty"` + // Use the incremental image build option, to reuse existing containers (default `true`) + IncrementalImageBuild *bool `property:"incremental-image-build" json:"incrementalImageBuild,omitempty"` // The build order strategy to use, either `dependencies`, `fifo` or `sequential` (default sequential) OrderStrategy string `property:"order-strategy" json:"orderStrategy,omitempty"` // When using `pod` strategy, the minimum amount of CPU required by the pod builder. diff --git a/pkg/apis/camel/v1/trait/zz_generated.deepcopy.go b/pkg/apis/camel/v1/trait/zz_generated.deepcopy.go index 10c0db3b93..3bfb466be0 100644 --- a/pkg/apis/camel/v1/trait/zz_generated.deepcopy.go +++ b/pkg/apis/camel/v1/trait/zz_generated.deepcopy.go @@ -64,6 +64,11 @@ func (in *BuilderTrait) DeepCopyInto(out *BuilderTrait) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.IncrementalImageBuild != nil { + in, out := &in.IncrementalImageBuild, &out.IncrementalImageBuild + *out = new(bool) + **out = **in + } if in.MavenProfiles != nil { in, out := &in.MavenProfiles, &out.MavenProfiles *out = make([]string, len(*in)) diff --git a/pkg/trait/builder.go b/pkg/trait/builder.go index 82ba4ebcb5..bd8bad40cd 100644 --- a/pkg/trait/builder.go +++ b/pkg/trait/builder.go @@ -34,6 +34,10 @@ import ( "github.com/apache/camel-k/v2/pkg/util/property" ) +const ( + builderTraitID = "builder" +) + var commandsRegexp = regexp.MustCompile(`"[^"]+"|[\w/-]+`) type builderTrait struct { @@ -43,7 +47,7 @@ type builderTrait struct { func newBuilderTrait() Trait { return &builderTrait{ - BaseTrait: NewBaseTrait("builder", 600), + BaseTrait: NewBaseTrait(builderTraitID, 600), } } diff --git a/pkg/trait/quarkus.go b/pkg/trait/quarkus.go index c1fd3cdac5..a342af77fd 100644 --- a/pkg/trait/quarkus.go +++ b/pkg/trait/quarkus.go @@ -366,7 +366,11 @@ func (t *quarkusTrait) applyWhenBuildSubmitted(e *Environment) error { // Default, if nothing is specified buildTask.Maven.Properties["quarkus.package.type"] = string(fastJarPackageType) packageSteps = append(packageSteps, builder.Quarkus.ComputeQuarkusDependencies) - packageSteps = append(packageSteps, builder.Image.IncrementalImageContext) + if t.isIncrementalImageBuild(e) { + packageSteps = append(packageSteps, builder.Image.IncrementalImageContext) + } else { + packageSteps = append(packageSteps, builder.Image.StandardImageContext) + } // Create the dockerfile, regardless it's later used or not by the publish strategy packageSteps = append(packageSteps, builder.Image.JvmDockerfile) } @@ -396,6 +400,17 @@ func (t *quarkusTrait) isNativeKit(e *Environment) (bool, error) { } } +func (t *quarkusTrait) isIncrementalImageBuild(e *Environment) bool { + // We need to get this information from the builder trait + if trait := e.Catalog.GetTrait(builderTraitID); trait != nil { + builder, ok := trait.(*builderTrait) + return ok && pointer.BoolDeref(builder.IncrementalImageBuild, true) + } + + // Default always to true for performance reasons + return true +} + func (t *quarkusTrait) applyWhenKitReady(e *Environment) error { if e.IntegrationInRunningPhases() && t.isNativeIntegration(e) { container := e.GetIntegrationContainer() diff --git a/pkg/trait/quarkus_test.go b/pkg/trait/quarkus_test.go index 331eb8c132..ca8ca396cd 100644 --- a/pkg/trait/quarkus_test.go +++ b/pkg/trait/quarkus_test.go @@ -27,6 +27,7 @@ import ( v1 "github.com/apache/camel-k/v2/pkg/apis/camel/v1" "github.com/apache/camel-k/v2/pkg/builder" "github.com/apache/camel-k/v2/pkg/util/camel" + "github.com/apache/camel-k/v2/pkg/util/test" ) func TestConfigureQuarkusTraitBuildSubmitted(t *testing.T) { @@ -95,8 +96,9 @@ func TestApplyQuarkusTraitAnnotationKitConfiguration(t *testing.T) { func createNominalQuarkusTest() (*quarkusTrait, *Environment) { trait, _ := newQuarkusTrait().(*quarkusTrait) trait.Enabled = pointer.Bool(true) - + client, _ := test.NewFakeClient() environment := &Environment{ + Catalog: NewCatalog(client), CamelCatalog: &camel.RuntimeCatalog{}, Integration: &v1.Integration{ Spec: v1.IntegrationSpec{ diff --git a/resources/traits.yaml b/resources/traits.yaml index 8b711afbcf..ca60f9824c 100755 --- a/resources/traits.yaml +++ b/resources/traits.yaml @@ -245,6 +245,10 @@ traits: - name: strategy type: string description: The strategy to use, either `pod` or `routine` (default routine) + - name: incremental-image-build + type: bool + description: Use the incremental image build option, to reuse existing containers + (default `true`) - name: order-strategy type: string description: The build order strategy to use, either `dependencies`, `fifo` or