diff --git a/.circleci/config.yml b/.circleci/config.yml index d174d798c3..d754539655 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ orbs: parameters: go-version: type: string - default: '1.22.7' + default: '1.23.0' executors: node: diff --git a/CHANGELOG.md b/CHANGELOG.md index 1754e8e84f..76874f4d41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # SingularityCE Changelog +## Changes Since Last Release + +### Bug Fixes + +- Fix regression from 4.1.5 that overwrites source image runscript, environment + etc. in build from local image. + ## 4.2.1 \[2024-09-13\] ### Bug Fixes diff --git a/e2e/build/regressions.go b/e2e/build/regressions.go index eafcd6d911..3d5db3c3fb 100644 --- a/e2e/build/regressions.go +++ b/e2e/build/regressions.go @@ -637,14 +637,35 @@ func (c *imgBuildTests) issue2607(t *testing.T) { } // Check that the build process from an image doesn't fail when the source image -// includes symlinks. +// includes symlinks. Also confirm that the base environment files from the +// source image are not overwritten (#3353). func (c *imgBuildTests) issue3084(t *testing.T) { + e2e.EnsureImage(t, c.env) require.Command(t, "mksquashfs") + // Extract standard test image to a sandbox dir. rootfs := filepath.Join(c.env.TestDir, "issue_3084_rootfs") if err := os.Mkdir(rootfs, 0o755); err != nil { t.Fatal(err) } + c.env.RunSingularity( + t, + e2e.WithProfile(e2e.UserProfile), + e2e.WithCommand("build"), + e2e.WithArgs("--force", "--sandbox", rootfs, c.env.ImagePath), + e2e.ExpectExit( + 0, + ), + ) + + // Remove existing `/tmp`, `/var`. Create `/var/tmp` -> `/tmp` and `/var/log` -> + // `/tmp` symlinks that cause unsquashfs extraction issue. + if err := os.RemoveAll(filepath.Join(rootfs, "tmp")); err != nil { + t.Fatal(err) + } + if err := os.RemoveAll(filepath.Join(rootfs, "var")); err != nil { + t.Fatal(err) + } if err := os.Mkdir(filepath.Join(rootfs, "tmp"), 0o755); err != nil { t.Fatal(err) } @@ -657,24 +678,41 @@ func (c *imgBuildTests) issue3084(t *testing.T) { if err := os.Symlink(filepath.Join(rootfs, "tmp"), filepath.Join(rootfs, "var", "log")); err != nil { t.Fatal(err) } + + // Build from resulting structure as a squashfs image := filepath.Join(c.env.TestDir, "issue_3084.img") if err := squashfs.Mksquashfs([]string{rootfs}, image); err != nil { t.Fatal(err) } - destImage := filepath.Join(c.env.TestDir, "issue_3084_dest.img") c.env.RunSingularity( t, e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("build"), e2e.WithArgs(destImage, image), + e2e.ExpectExit( + 0, + ), e2e.PostRun(func(_ *testing.T) { - os.Remove(destImage) os.Remove(image) os.RemoveAll(rootfs) }), + ) + + // https://github.com/sylabs/singularity/issues/3353 + // The source test image runscript outputs "Running command: $*" Make sure + // we see it... the runscript should still be in place. + c.env.RunSingularity( + t, + e2e.WithProfile(e2e.UserProfile), + e2e.WithCommand("run"), + e2e.WithArgs(destImage, "/bin/true"), e2e.ExpectExit( 0, + e2e.ExpectOutput(e2e.ContainMatch, "Running command: /bin/true"), ), + e2e.PostRun(func(_ *testing.T) { + os.Remove(destImage) + }), ) } diff --git a/internal/pkg/build/sources/base_environment.go b/internal/pkg/build/sources/base_environment.go index bd294535fd..4e1b6e7924 100644 --- a/internal/pkg/build/sources/base_environment.go +++ b/internal/pkg/build/sources/base_environment.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2021, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2024, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -343,7 +343,7 @@ func makeSymlinks(rootPath string) error { return nil } -func makeFile(name string, perm os.FileMode, s string) (err error) { +func makeFile(name string, perm os.FileMode, s string, overwrite bool) (err error) { // #4532 - If the file already exists ensure it has requested permissions // as OpenFile won't set on an existing file and some docker // containers have hosts or resolv.conf without write perm. @@ -351,6 +351,10 @@ func makeFile(name string, perm os.FileMode, s string) (err error) { if err = os.Chmod(name, perm); err != nil { return } + if !overwrite { + sylog.Debugf("not writing to %s - file exists, overwrite is false", name) + return + } } // Create the file if it's not in the container, or truncate and write s // into it otherwise. @@ -364,50 +368,55 @@ func makeFile(name string, perm os.FileMode, s string) (err error) { return } -func makeFiles(rootPath string) error { - if err := makeFile(filepath.Join(rootPath, "etc", "hosts"), 0o644, ""); err != nil { +func makeFiles(rootPath string, overwrite bool) error { + if err := makeFile(filepath.Join(rootPath, "etc", "hosts"), 0o644, "", overwrite); err != nil { return err } - if err := makeFile(filepath.Join(rootPath, "etc", "resolv.conf"), 0o644, ""); err != nil { + if err := makeFile(filepath.Join(rootPath, "etc", "resolv.conf"), 0o644, "", overwrite); err != nil { return err } - if err := makeFile(filepath.Join(rootPath, ".singularity.d", "actions", "exec"), 0o755, execFileContent); err != nil { + if err := makeFile(filepath.Join(rootPath, ".singularity.d", "actions", "exec"), 0o755, execFileContent, overwrite); err != nil { return err } - if err := makeFile(filepath.Join(rootPath, ".singularity.d", "actions", "run"), 0o755, runFileContent); err != nil { + if err := makeFile(filepath.Join(rootPath, ".singularity.d", "actions", "run"), 0o755, runFileContent, overwrite); err != nil { return err } - if err := makeFile(filepath.Join(rootPath, ".singularity.d", "actions", "shell"), 0o755, shellFileContent); err != nil { + if err := makeFile(filepath.Join(rootPath, ".singularity.d", "actions", "shell"), 0o755, shellFileContent, overwrite); err != nil { return err } - if err := makeFile(filepath.Join(rootPath, ".singularity.d", "actions", "start"), 0o755, startFileContent); err != nil { + if err := makeFile(filepath.Join(rootPath, ".singularity.d", "actions", "start"), 0o755, startFileContent, overwrite); err != nil { return err } - if err := makeFile(filepath.Join(rootPath, ".singularity.d", "actions", "test"), 0o755, testFileContent); err != nil { + if err := makeFile(filepath.Join(rootPath, ".singularity.d", "actions", "test"), 0o755, testFileContent, overwrite); err != nil { return err } - if err := makeFile(filepath.Join(rootPath, ".singularity.d", "env", "01-base.sh"), 0o755, baseShFileContent); err != nil { + if err := makeFile(filepath.Join(rootPath, ".singularity.d", "env", "01-base.sh"), 0o755, baseShFileContent, overwrite); err != nil { return err } - if err := makeFile(filepath.Join(rootPath, ".singularity.d", "env", "90-environment.sh"), 0o755, environmentShFileContent); err != nil { + if err := makeFile(filepath.Join(rootPath, ".singularity.d", "env", "90-environment.sh"), 0o755, environmentShFileContent, overwrite); err != nil { return err } - if err := makeFile(filepath.Join(rootPath, ".singularity.d", "env", "95-apps.sh"), 0o755, appsShFileContent); err != nil { + if err := makeFile(filepath.Join(rootPath, ".singularity.d", "env", "95-apps.sh"), 0o755, appsShFileContent, overwrite); err != nil { return err } - if err := makeFile(filepath.Join(rootPath, ".singularity.d", "env", "99-base.sh"), 0o755, base99ShFileContent); err != nil { + if err := makeFile(filepath.Join(rootPath, ".singularity.d", "env", "99-base.sh"), 0o755, base99ShFileContent, overwrite); err != nil { return err } - if err := makeFile(filepath.Join(rootPath, ".singularity.d", "env", "99-runtimevars.sh"), 0o755, base99runtimevarsShFileContent); err != nil { + if err := makeFile(filepath.Join(rootPath, ".singularity.d", "env", "99-runtimevars.sh"), 0o755, base99runtimevarsShFileContent, overwrite); err != nil { return err } - if err := makeFile(filepath.Join(rootPath, ".singularity.d", "runscript"), 0o755, runscriptFileContent); err != nil { + if err := makeFile(filepath.Join(rootPath, ".singularity.d", "runscript"), 0o755, runscriptFileContent, overwrite); err != nil { return err } - return makeFile(filepath.Join(rootPath, ".singularity.d", "startscript"), 0o755, startscriptFileContent) + return makeFile(filepath.Join(rootPath, ".singularity.d", "startscript"), 0o755, startscriptFileContent, overwrite) } -func makeBaseEnv(rootPath string) (err error) { +// makeBaseEnv inserts Singularity specific directories, symlinks, and files +// into the contatiner rootfs. If overwrite is true, then any existing files +// will be overwritten with new content. If overwrite is false, existing files +// (e.g. where the rootfs has been extracted from an existing image) will not be +// modified. +func makeBaseEnv(rootPath string, overwrite bool) (err error) { var info os.FileInfo // Ensure we can write into the root of rootPath @@ -431,7 +440,7 @@ func makeBaseEnv(rootPath string) (err error) { err = fmt.Errorf("build: failed to make environment symlinks: %v", err) return err } - if err = makeFiles(rootPath); err != nil { + if err = makeFiles(rootPath, overwrite); err != nil { err = fmt.Errorf("build: failed to make environment files: %v", err) return err } diff --git a/internal/pkg/build/sources/base_environment_test.go b/internal/pkg/build/sources/base_environment_test.go index 1324b77d4e..b4f6c943bd 100644 --- a/internal/pkg/build/sources/base_environment_test.go +++ b/internal/pkg/build/sources/base_environment_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2024, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -49,9 +49,9 @@ func TestMakeFiles(t *testing.T) { if err := makeDirs(d); err != nil { return err } - return makeFiles(d) + return makeFiles(d, false) }) - testWithBadDir(t, makeFiles) + testWithBadDir(t, func(d string) error { return makeFiles(d, false) }) // #4532 - Check that we can succeed with an existing file that doesn't have // write permission. testWithGoodDir(t, func(d string) error { @@ -62,7 +62,7 @@ func TestMakeFiles(t *testing.T) { if err != nil { t.Fatalf("Failed to make test hosts file: %s", err) } - return makeFiles(d) + return makeFiles(d, false) }) } @@ -70,6 +70,6 @@ func TestMakeBaseEnv(t *testing.T) { test.DropPrivilege(t) defer test.ResetPrivilege(t) - testWithGoodDir(t, makeBaseEnv) - testWithBadDir(t, makeBaseEnv) + testWithGoodDir(t, func(d string) error { return makeBaseEnv(d, false) }) + testWithBadDir(t, func(d string) error { return makeBaseEnv(d, false) }) } diff --git a/internal/pkg/build/sources/conveyorPacker_arch.go b/internal/pkg/build/sources/conveyorPacker_arch.go index 75d0a9df51..7f038ec756 100644 --- a/internal/pkg/build/sources/conveyorPacker_arch.go +++ b/internal/pkg/build/sources/conveyorPacker_arch.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2024, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -204,7 +204,7 @@ func (cp *ArchConveyorPacker) getPacConf(pacmanConfURL string) (pacConf string, } func (cp *ArchConveyorPacker) insertBaseEnv() (err error) { - if err = makeBaseEnv(cp.b.RootfsPath); err != nil { + if err = makeBaseEnv(cp.b.RootfsPath, true); err != nil { return } return nil diff --git a/internal/pkg/build/sources/conveyorPacker_busybox.go b/internal/pkg/build/sources/conveyorPacker_busybox.go index 95d1ad0eb1..773365a087 100644 --- a/internal/pkg/build/sources/conveyorPacker_busybox.go +++ b/internal/pkg/build/sources/conveyorPacker_busybox.go @@ -121,7 +121,7 @@ func (c *BusyBoxConveyor) insertBusyBox(mirrorurl string) (busyBoxPath string, e } func (c *BusyBoxConveyor) insertBaseEnv() (err error) { - if err = makeBaseEnv(c.b.RootfsPath); err != nil { + if err = makeBaseEnv(c.b.RootfsPath, true); err != nil { return } return nil diff --git a/internal/pkg/build/sources/conveyorPacker_debootstrap.go b/internal/pkg/build/sources/conveyorPacker_debootstrap.go index 37286fe957..9af48c39be 100644 --- a/internal/pkg/build/sources/conveyorPacker_debootstrap.go +++ b/internal/pkg/build/sources/conveyorPacker_debootstrap.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2024, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -267,7 +267,7 @@ func (cp *DebootstrapConveyorPacker) getRecipeHeaderInfo() (err error) { } func (cp *DebootstrapConveyorPacker) insertBaseEnv(b *types.Bundle) (err error) { - if err = makeBaseEnv(b.RootfsPath); err != nil { + if err = makeBaseEnv(b.RootfsPath, true); err != nil { return } return nil diff --git a/internal/pkg/build/sources/conveyorPacker_library.go b/internal/pkg/build/sources/conveyorPacker_library.go index 570c30dfc2..4828542a59 100644 --- a/internal/pkg/build/sources/conveyorPacker_library.go +++ b/internal/pkg/build/sources/conveyorPacker_library.go @@ -1,5 +1,5 @@ // Copyright (c) 2020, Control Command Inc. All rights reserved. -// Copyright (c) 2018-2023, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2024, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -38,10 +38,6 @@ func (cp *LibraryConveyorPacker) Get(ctx context.Context, b *types.Bundle) (err libraryURL := b.Opts.LibraryURL authToken := b.Opts.LibraryAuthToken - if err = makeBaseEnv(cp.b.RootfsPath); err != nil { - return fmt.Errorf("while inserting base environment: %v", err) - } - // check for custom library from definition customLib, ok := b.Recipe.Header["library"] if ok { @@ -82,7 +78,7 @@ func (cp *LibraryConveyorPacker) Get(ctx context.Context, b *types.Bundle) (err } // insert base metadata before unpacking fs - if err = makeBaseEnv(cp.b.RootfsPath); err != nil { + if err = makeBaseEnv(cp.b.RootfsPath, true); err != nil { return fmt.Errorf("while inserting base environment: %v", err) } diff --git a/internal/pkg/build/sources/conveyorPacker_local.go b/internal/pkg/build/sources/conveyorPacker_local.go index 9435d516cd..27a2946f5c 100644 --- a/internal/pkg/build/sources/conveyorPacker_local.go +++ b/internal/pkg/build/sources/conveyorPacker_local.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2024, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -112,9 +112,11 @@ func (cp *LocalConveyorPacker) Pack(ctx context.Context) (*types.Bundle, error) return nil, fmt.Errorf("while unpacking local image: %v", err) } - // insert base metadata AFTER unpacking fs to avoid conflicts with contained files/symlinks + // Insert base metadata after unpacking fs to avoid unsquashfs failure on + // existing files/symlink. Call makeBaseEnv with overwrite=false so we don't + // overwrite runscripts etc. that were extracted from the image. sylog.Infof("Inserting Singularity configuration...") - if err = makeBaseEnv(b.RootfsPath); err != nil { + if err = makeBaseEnv(b.RootfsPath, false); err != nil { return nil, fmt.Errorf("while inserting base environment: %v", err) } diff --git a/internal/pkg/build/sources/conveyorPacker_oci.go b/internal/pkg/build/sources/conveyorPacker_oci.go index e825075998..c5725ad9ea 100644 --- a/internal/pkg/build/sources/conveyorPacker_oci.go +++ b/internal/pkg/build/sources/conveyorPacker_oci.go @@ -247,7 +247,7 @@ func (cp *OCIConveyorPacker) unpackRootfs(ctx context.Context) error { } func (cp *OCIConveyorPacker) insertBaseEnv() (err error) { - if err = makeBaseEnv(cp.b.RootfsPath); err != nil { + if err = makeBaseEnv(cp.b.RootfsPath, true); err != nil { sylog.Errorf("%v", err) } return diff --git a/internal/pkg/build/sources/conveyorPacker_oras.go b/internal/pkg/build/sources/conveyorPacker_oras.go index fff2a5b6d3..11912ce618 100644 --- a/internal/pkg/build/sources/conveyorPacker_oras.go +++ b/internal/pkg/build/sources/conveyorPacker_oras.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2020-2024, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -45,7 +45,7 @@ func (cp *OrasConveyorPacker) Get(ctx context.Context, b *types.Bundle) (err err } // insert base metadata before unpacking fs - if err = makeBaseEnv(b.RootfsPath); err != nil { + if err = makeBaseEnv(b.RootfsPath, true); err != nil { return fmt.Errorf("while inserting base environment: %v", err) } diff --git a/internal/pkg/build/sources/conveyorPacker_scratch.go b/internal/pkg/build/sources/conveyorPacker_scratch.go index d8e35ca79a..cd5a1a447f 100644 --- a/internal/pkg/build/sources/conveyorPacker_scratch.go +++ b/internal/pkg/build/sources/conveyorPacker_scratch.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2024, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -47,7 +47,7 @@ func (cp *ScratchConveyorPacker) Pack(context.Context) (b *types.Bundle, err err } func (c *ScratchConveyor) insertBaseEnv() (err error) { - if err = makeBaseEnv(c.b.RootfsPath); err != nil { + if err = makeBaseEnv(c.b.RootfsPath, true); err != nil { return } return nil diff --git a/internal/pkg/build/sources/conveyorPacker_shub.go b/internal/pkg/build/sources/conveyorPacker_shub.go index 3e71c1a076..c82fa3ed94 100644 --- a/internal/pkg/build/sources/conveyorPacker_shub.go +++ b/internal/pkg/build/sources/conveyorPacker_shub.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2024, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -34,7 +34,7 @@ func (cp *ShubConveyorPacker) Get(ctx context.Context, b *types.Bundle) (err err } // insert base metadata before unpacking fs - if err = makeBaseEnv(cp.b.RootfsPath); err != nil { + if err = makeBaseEnv(cp.b.RootfsPath, true); err != nil { return fmt.Errorf("while inserting base environment: %v", err) } diff --git a/internal/pkg/build/sources/conveyorPacker_yum.go b/internal/pkg/build/sources/conveyorPacker_yum.go index c4834586c4..7a804db7f1 100644 --- a/internal/pkg/build/sources/conveyorPacker_yum.go +++ b/internal/pkg/build/sources/conveyorPacker_yum.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2024, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -339,7 +339,7 @@ func (c *YumConveyor) makePseudoDevices() (err error) { } func (cp *YumConveyorPacker) insertBaseEnv() (err error) { - if err = makeBaseEnv(cp.b.RootfsPath); err != nil { + if err = makeBaseEnv(cp.b.RootfsPath, true); err != nil { return } return nil diff --git a/internal/pkg/build/sources/conveyorPacker_zypper.go b/internal/pkg/build/sources/conveyorPacker_zypper.go index 58649c3973..682e527565 100644 --- a/internal/pkg/build/sources/conveyorPacker_zypper.go +++ b/internal/pkg/build/sources/conveyorPacker_zypper.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2024, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -361,7 +361,7 @@ func (cp *ZypperConveyorPacker) Pack(context.Context) (b *types.Bundle, err erro } func (cp *ZypperConveyorPacker) insertBaseEnv() (err error) { - if err = makeBaseEnv(cp.b.RootfsPath); err != nil { + if err = makeBaseEnv(cp.b.RootfsPath, true); err != nil { return } return nil