From 60eefabb20063639fec2e346d4a38a05d886c375 Mon Sep 17 00:00:00 2001 From: Florian Wilhelm Date: Mon, 22 Jun 2020 10:39:52 +0200 Subject: [PATCH] Install dev deps in mta build (#1685) Ensure npm dev dependencies are available after mtaBuild as they are required by certain tests. Co-authored-by: Kevin Hudemann Co-authored-by: Daniel Kurzynski --- cmd/mtaBuild.go | 14 +++- cmd/mtaBuild_generated.go | 10 +++ integration/integration_mta_build_test.go | 69 ++++++++++++++++++- .../TestMtaIntegration/maven/.gitignore | 2 + .../TestMtaIntegration/maven/app/package.json | 10 +++ .../npm-install-dev-dependencies/.gitignore | 4 ++ .../npm-install-dev-dependencies/cleanup.sh | 4 ++ .../npm-install-dev-dependencies/mta.yaml | 8 +++ .../npm-install-dev-dependencies/package.json | 10 +++ resources/metadata/mtaBuild.yaml | 8 +++ 10 files changed, 136 insertions(+), 3 deletions(-) create mode 100644 integration/testdata/TestMtaIntegration/maven/app/package.json create mode 100644 integration/testdata/TestMtaIntegration/npm-install-dev-dependencies/.gitignore create mode 100755 integration/testdata/TestMtaIntegration/npm-install-dev-dependencies/cleanup.sh create mode 100644 integration/testdata/TestMtaIntegration/npm-install-dev-dependencies/mta.yaml create mode 100644 integration/testdata/TestMtaIntegration/npm-install-dev-dependencies/package.json diff --git a/cmd/mtaBuild.go b/cmd/mtaBuild.go index c8381eb6f7..784fe68137 100644 --- a/cmd/mtaBuild.go +++ b/cmd/mtaBuild.go @@ -190,8 +190,18 @@ func runMtaBuild(config mtaBuildOptions, commonPipelineEnvironment.mtarFilePath = mtarName - err = installMavenArtifacts(e, config) - + if config.InstallArtifacts { + // install maven artifacts in local maven repo because `mbt build` executes `mvn package -B` + err = installMavenArtifacts(e, config) + if err != nil { + return err + } + // mta-builder executes 'npm install --production', therefore we need 'npm ci/install' to install the dev-dependencies + err = npmExecutor.InstallAllDependencies(npmExecutor.FindPackageJSONFiles()) + if err != nil { + return err + } + } return err } diff --git a/cmd/mtaBuild_generated.go b/cmd/mtaBuild_generated.go index 26fd8d3cf5..9fd1ff9558 100644 --- a/cmd/mtaBuild_generated.go +++ b/cmd/mtaBuild_generated.go @@ -28,6 +28,7 @@ type mtaBuildOptions struct { ProjectSettingsFile string `json:"projectSettingsFile,omitempty"` GlobalSettingsFile string `json:"globalSettingsFile,omitempty"` M2Path string `json:"m2Path,omitempty"` + InstallArtifacts bool `json:"installArtifacts,omitempty"` } type mtaBuildCommonPipelineEnvironment struct { @@ -124,6 +125,7 @@ func addMtaBuildFlags(cmd *cobra.Command, stepConfig *mtaBuildOptions) { cmd.Flags().StringVar(&stepConfig.ProjectSettingsFile, "projectSettingsFile", os.Getenv("PIPER_projectSettingsFile"), "Path or url to the mvn settings file that should be used as project settings file.") cmd.Flags().StringVar(&stepConfig.GlobalSettingsFile, "globalSettingsFile", os.Getenv("PIPER_globalSettingsFile"), "Path or url to the mvn settings file that should be used as global settings file") cmd.Flags().StringVar(&stepConfig.M2Path, "m2Path", os.Getenv("PIPER_m2Path"), "Path to the location of the local repository that should be used.") + cmd.Flags().BoolVar(&stepConfig.InstallArtifacts, "installArtifacts", false, "If enabled, for npm packages this step will install all depedencies including dev dependencies. For maven it will install all artifacts to the local maven repository.") } @@ -233,6 +235,14 @@ func mtaBuildMetadata() config.StepData { Mandatory: false, Aliases: []config.Alias{{Name: "maven/m2Path"}}, }, + { + Name: "installArtifacts", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"GENERAL", "STEPS", "STAGES", "PARAMETERS"}, + Type: "bool", + Mandatory: false, + Aliases: []config.Alias{}, + }, }, }, }, diff --git a/integration/integration_mta_build_test.go b/integration/integration_mta_build_test.go index 8e4e020227..218fa1ea48 100644 --- a/integration/integration_mta_build_test.go +++ b/integration/integration_mta_build_test.go @@ -43,9 +43,11 @@ cd /test apt-get -yqq update; apt-get -yqq install make curl -OL https://github.com/SAP/cloud-mta-build-tool/releases/download/v1.0.14/cloud-mta-build-tool_1.0.14_Linux_amd64.tar.gz tar xzf cloud-mta-build-tool_1.0.14_Linux_amd64.tar.gz +curl -sL https://deb.nodesource.com/setup_12.x | bash - +apt-get install -yqq nodejs mv mbt /usr/bin mkdir mym2 -/piperbin/piper mtaBuild --m2Path=mym2 >test-log.txt 2>&1 +/piperbin/piper mtaBuild --installArtifacts --m2Path=mym2 >test-log.txt 2>&1 ` ioutil.WriteFile(filepath.Join(tempDir, "runPiper.sh"), []byte(testScript), 0700) @@ -79,6 +81,7 @@ mkdir mym2 assert.Contains(t, output, "Installing /test/.flattened-pom.xml to /test/mym2/mygroup/mymvn/1.0-SNAPSHOT/mymvn-1.0-SNAPSHOT.pom") assert.Contains(t, output, "Installing /test/app/target/mymvn-app-1.0-SNAPSHOT.war to /test/mym2/mygroup/mymvn-app/1.0-SNAPSHOT/mymvn-app-1.0-SNAPSHOT.war") assert.Contains(t, output, "Installing /test/app/target/mymvn-app-1.0-SNAPSHOT-classes.jar to /test/mym2/mygroup/mymvn-app/1.0-SNAPSHOT/mymvn-app-1.0-SNAPSHOT-classes.jar") + assert.Contains(t, output, "added 2 packages from 3 contributors and audited 2 packages in") } func TestNPMProject(t *testing.T) { @@ -144,3 +147,67 @@ mv mbt /usr/bin output := string(content) assert.Contains(t, output, "INFO the MTA archive generated at: test-mta-js.mtar") } + +func TestNPMProjectInstallsDevDependencies(t *testing.T) { + t.Parallel() + ctx := context.Background() + + pwd, err := os.Getwd() + if err != nil { + t.Fatalf("Getting current working directory failed: %v", err) + } + pwd = filepath.Dir(pwd) + + // using custom createTmpDir function to avoid issues with symlinks on Docker for Mac + tempDir, err := createTmpDir("") + defer os.RemoveAll(tempDir) // clean up + + if err != nil { + t.Fatalf("Error when creating temp dir: %v", err) + } + + err = copyDir(filepath.Join(pwd, "integration", "testdata", "TestMtaIntegration", "npm-install-dev-dependencies"), tempDir) + if err != nil { + t.Fatal("Failed to copy test project.") + } + + //workaround to use test script util it is possible to set workdir for Exec call + testScript := `#!/bin/sh +cd /test +apt-get -yqq update; apt-get -yqq install make +curl -OL https://github.com/SAP/cloud-mta-build-tool/releases/download/v1.0.14/cloud-mta-build-tool_1.0.14_Linux_amd64.tar.gz +tar xzf cloud-mta-build-tool_1.0.14_Linux_amd64.tar.gz +mv mbt /usr/bin +/piperbin/piper mtaBuild --installArtifacts >test-log.txt 2>&1 +` + ioutil.WriteFile(filepath.Join(tempDir, "runPiper.sh"), []byte(testScript), 0700) + + reqNode := testcontainers.ContainerRequest{ + Image: "node:12", + Cmd: []string{"tail", "-f"}, + + BindMounts: map[string]string{ + pwd: "/piperbin", + tempDir: "/test", + }, + } + + mbtContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: reqNode, + Started: true, + }) + + code, err := mbtContainer.Exec(ctx, []string{"sh", "/test/runPiper.sh"}) + + if err != nil { + t.Fatalf("Script returened error: %v", err) + } + assert.Equal(t, 0, code) + + content, err := ioutil.ReadFile(filepath.Join(tempDir, "/test-log.txt")) + if err != nil { + t.Fatal("Could not read test-log.txt.", err) + } + output := string(content) + assert.Contains(t, output, "added 2 packages in") +} diff --git a/integration/testdata/TestMtaIntegration/maven/.gitignore b/integration/testdata/TestMtaIntegration/maven/.gitignore index 72f98cb55b..fcd6ccc8b5 100644 --- a/integration/testdata/TestMtaIntegration/maven/.gitignore +++ b/integration/testdata/TestMtaIntegration/maven/.gitignore @@ -5,3 +5,5 @@ foo.mtar .flattened-pom.xml .pipeline/ mym2 +app/node_modules +app/package-lock.json diff --git a/integration/testdata/TestMtaIntegration/maven/app/package.json b/integration/testdata/TestMtaIntegration/maven/app/package.json new file mode 100644 index 0000000000..58b4551ec6 --- /dev/null +++ b/integration/testdata/TestMtaIntegration/maven/app/package.json @@ -0,0 +1,10 @@ +{ + "name": "my-awesome-package", + "version": "1.0.0", + "dependencies": { + "uuid": "^8.1.0" + }, + "devDependencies": { + "is-number": "^7.0.0" + } +} diff --git a/integration/testdata/TestMtaIntegration/npm-install-dev-dependencies/.gitignore b/integration/testdata/TestMtaIntegration/npm-install-dev-dependencies/.gitignore new file mode 100644 index 0000000000..7967bcc3ee --- /dev/null +++ b/integration/testdata/TestMtaIntegration/npm-install-dev-dependencies/.gitignore @@ -0,0 +1,4 @@ +.pipeline +node_modules +package-lock.json +test-mta-js.mtar diff --git a/integration/testdata/TestMtaIntegration/npm-install-dev-dependencies/cleanup.sh b/integration/testdata/TestMtaIntegration/npm-install-dev-dependencies/cleanup.sh new file mode 100755 index 0000000000..ce0cdd9c8a --- /dev/null +++ b/integration/testdata/TestMtaIntegration/npm-install-dev-dependencies/cleanup.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC2002 +cat .gitignore | xargs -L1 rm -r diff --git a/integration/testdata/TestMtaIntegration/npm-install-dev-dependencies/mta.yaml b/integration/testdata/TestMtaIntegration/npm-install-dev-dependencies/mta.yaml new file mode 100644 index 0000000000..370d89a4a4 --- /dev/null +++ b/integration/testdata/TestMtaIntegration/npm-install-dev-dependencies/mta.yaml @@ -0,0 +1,8 @@ +_schema-version: '3.1' +ID: test-mta-js +version: 1.0.0 + +modules: + - name: test-mta-js-srv + type: nodejs + path: . diff --git a/integration/testdata/TestMtaIntegration/npm-install-dev-dependencies/package.json b/integration/testdata/TestMtaIntegration/npm-install-dev-dependencies/package.json new file mode 100644 index 0000000000..58b4551ec6 --- /dev/null +++ b/integration/testdata/TestMtaIntegration/npm-install-dev-dependencies/package.json @@ -0,0 +1,10 @@ +{ + "name": "my-awesome-package", + "version": "1.0.0", + "dependencies": { + "uuid": "^8.1.0" + }, + "devDependencies": { + "is-number": "^7.0.0" + } +} diff --git a/resources/metadata/mtaBuild.yaml b/resources/metadata/mtaBuild.yaml index 860727571e..65e902241f 100644 --- a/resources/metadata/mtaBuild.yaml +++ b/resources/metadata/mtaBuild.yaml @@ -129,6 +129,14 @@ spec: mandatory: false aliases: - name: maven/m2Path + - name: installArtifacts + type: bool + description: If enabled, for npm packages this step will install all depedencies including dev dependencies. For maven it will install all artifacts to the local maven repository. + scope: + - GENERAL + - STEPS + - STAGES + - PARAMETERS outputs: resources: - name: commonPipelineEnvironment