bib: run "dnf" inside the container again

.github/workflows/testingfarm-unit.yml

@@ -0,0 +1,43 @@
+---
+name: Testing farm go unit tests
+
+on:
+  pull_request_target:
+    types: [opened, synchronize, reopened]
+
+# see testingfarm.yml
+jobs:
+  testingfarm:
+    name: "Run in testing farm"
+    runs-on: ubuntu-latest
+    steps:
+    - name: Get User Permission
+      id: checkAccess
+      uses: actions-cool/check-user-permission@v2
+      with:
+        require: write
+        username: ${{ github.triggering_actor }}
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - name: Check User Permission
+      if: steps.checkAccess.outputs.require-result == 'false'
+      run: |
+        echo "${{ github.triggering_actor }} does not have permissions on this repo."
+        echo "Current permission level is ${{ steps.checkAccess.outputs.user-permission }}"
+        echo "Job originally triggered by ${{ github.actor }}"
+        exit 1
+    - name: Check out code
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+    - name: Run the tests
+      uses: sclorg/testing-farm-as-github-action@v3
+      with:
+        compose: Fedora-40
+        tmt_plan_filter: "unit-go.fmf"
+        api_key: ${{ secrets.TF_API_KEY }}
+        git_url: ${{ github.event.pull_request.head.repo.clone_url }}
+        git_ref: ${{ github.event.pull_request.head.ref }}
+        pull_request_status_name: "Testing farm"
+        tf_scope: private
+        secrets: "AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }};AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }};RHSM_ORG=${{ secrets.RHSM_ORG }};RHSM_ACTIVATION_KEY=${{ secrets.RHSM_ACTIVATION_KEY }} "

.github/workflows/testingfarm.yml

@@ -51,9 +51,10 @@ jobs:
       uses: sclorg/testing-farm-as-github-action@v3
       with:
         compose: Fedora-40
+        tmt_plan_filter: "integration.fmf"
         api_key: ${{ secrets.TF_API_KEY }}
         git_url: ${{ github.event.pull_request.head.repo.clone_url }}
         git_ref: ${{ github.event.pull_request.head.ref }}
         pull_request_status_name: "Testing farm"
         tf_scope: private
-        secrets: "AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }};AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}"
+        secrets: "AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }};AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }};RHSM_ORG=${{ secrets.RHSM_ORG }};RHSM_ACTIVATION_KEY=${{ secrets.RHSM_ACTIVATION_KEY }} "

bib/cmd/bootc-image-builder/image.go

@@ -60,9 +60,6 @@ type ManifestConfig struct {
 	// Extracted information about the source container image
 	SourceInfo *source.Info
 
-	// Path to the tree that contains /etc used for osbuild-depsolve-dnf
-	DepsolverRootDir string
-
 	// RootFSType specifies the filesystem type for the root partition
 	RootFSType string
 }

bib/cmd/bootc-image-builder/main.go

@@ -102,20 +102,13 @@ func getContainerSize(imgref string) (uint64, error) {
 	return size, nil
 }
 
-func makeManifest(c *ManifestConfig, cacheRoot string) (manifest.OSBuildManifest, map[string][]rpmmd.RepoConfig, error) {
+func makeManifest(c *ManifestConfig, solver *dnfjson.Solver, cacheRoot string) (manifest.OSBuildManifest, map[string][]rpmmd.RepoConfig, error) {
 	manifest, err := Manifest(c)
 	if err != nil {
 		return nil, nil, fmt.Errorf("cannot get manifest: %w", err)
 	}
 
 	// depsolve packages
-	solver := dnfjson.NewSolver(
-		c.SourceInfo.OSRelease.PlatformID,
-		c.SourceInfo.OSRelease.VersionID,
-		c.Architecture.String(),
-		fmt.Sprintf("%s-%s", c.SourceInfo.OSRelease.ID, c.SourceInfo.OSRelease.VersionID),
-		cacheRoot)
-	solver.SetRootDir(c.DepsolverRootDir)
 	depsolvedSets := make(map[string][]rpmmd.PackageSpec)
 	depsolvedRepos := make(map[string][]rpmmd.RepoConfig)
 	for name, pkgSet := range manifest.GetPackageSetChains() {
@@ -285,32 +278,35 @@ func manifestFromCobra(cmd *cobra.Command, args []string) ([]byte, *mTLSConfig,
 			rootfsType = "ext4"
 		}
 	}
+	// Gather some data from the containers distro
+	sourceinfo, err := source.LoadInfo(container.Root())
+	if err != nil {
+		return nil, nil, err
+	}
 
 	// This is needed just for RHEL and RHSM in most cases, but let's run it every time in case
 	// the image has some non-standard dnf plugins.
 	if err := container.InitDNF(); err != nil {
 		return nil, nil, err
 	}
-
-	sourceinfo, err := source.LoadInfo(container.Root())
+	solver, err := container.NewContainerSolver(rpmCacheRoot, cntArch, sourceinfo)
 	if err != nil {
 		return nil, nil, err
 	}
 
 	manifestConfig := &ManifestConfig{
-		Architecture:     cntArch,
-		Config:           config,
-		ImageTypes:       imageTypes,
-		Imgref:           imgref,
-		TLSVerify:        tlsVerify,
-		RootfsMinsize:    cntSize * containerSizeToDiskSizeMultiplier,
-		DistroDefPaths:   distroDefPaths,
-		SourceInfo:       sourceinfo,
-		RootFSType:       rootfsType,
-		DepsolverRootDir: container.Root(),
-	}
-
-	manifest, repos, err := makeManifest(manifestConfig, rpmCacheRoot)
+		Architecture:   cntArch,
+		Config:         config,
+		ImageTypes:     imageTypes,
+		Imgref:         imgref,
+		TLSVerify:      tlsVerify,
+		RootfsMinsize:  cntSize * containerSizeToDiskSizeMultiplier,
+		DistroDefPaths: distroDefPaths,
+		SourceInfo:     sourceinfo,
+		RootFSType:     rootfsType,
+	}
+
+	manifest, repos, err := makeManifest(manifestConfig, solver, rpmCacheRoot)
 	if err != nil {
 		return nil, nil, err
 	}

bib/internal/container/container.go

@@ -123,26 +123,6 @@ func (c *Container) ExecArgv() []string {
 	return []string{"podman", "exec", "-i", c.id}
 }
 
-// InitDNF initializes dnf in the container. This is necessary when
-// the caller wants to read the image's dnf repositories, but they are
-// not static, but rather configured by dnf dynamically. The primaru
-// use-case for this is RHEL and subscription-manager.
-//
-// The implementation is simple: We just run plain `dnf` in the
-// container so that the subscription-manager gets initialized. For
-// compatibility with both dnf and dnf5 we cannot just run "dnf" as
-// dnf5 will error and do nothing in this case. So we use "dnf check
-// --duplicates" as this is fast on both dnf4/dnf5 (just doing "dnf5
-// check" without arguments takes around 25s so that is not a great
-// option).
-func (c *Container) InitDNF() error {
-	if output, err := exec.Command("podman", "exec", c.id, "dnf", "check", "--duplicates").CombinedOutput(); err != nil {
-		return fmt.Errorf("initializing dnf in %s container failed: %w\noutput:\n%s", c.id, err, string(output))
-	}
-
-	return nil
-}
-
 // DefaultRootfsType returns the default rootfs type (e.g. "ext4") as
 // specified by the bootc container install configuration. An empty
 // string is valid and means the container sets no default.

bib/internal/container/solver.go

@@ -0,0 +1,68 @@
+package container
+
+import (
+	"fmt"
+	"os/exec"
+	"path/filepath"
+
+	"github.com/osbuild/images/pkg/arch"
+	"github.com/osbuild/images/pkg/dnfjson"
+
+	"github.com/osbuild/bootc-image-builder/bib/internal/source"
+)
+
+// InitDNF initializes dnf in the container. This is necessary when
+// the caller wants to read the image's dnf repositories, but they are
+// not static, but rather configured by dnf dynamically. The primaru
+// use-case for this is RHEL and subscription-manager.
+//
+// The implementation is simple: We just run plain `dnf` in the
+// container so that the subscription-manager gets initialized. For
+// compatibility with both dnf and dnf5 we cannot just run "dnf" as
+// dnf5 will error and do nothing in this case. So we use "dnf check
+// --duplicates" as this is fast on both dnf4/dnf5 (just doing "dnf5
+// check" without arguments takes around 25s so that is not a great
+// option).
+func (c *Container) InitDNF() error {
+	if output, err := exec.Command("podman", "exec", c.id, "dnf", "check", "--duplicates").CombinedOutput(); err != nil {
+		return fmt.Errorf("initializing dnf in %s container failed: %w\noutput:\n%s", c.id, err, string(output))
+	}
+
+	return nil
+}
+
+func (cnt *Container) injectDNFJson() ([]string, error) {
+	if err := cnt.CopyInto("/usr/libexec/osbuild-depsolve-dnf", "/osbuild-depsolve-dnf"); err != nil {
+		return nil, fmt.Errorf("cannot prepare depsolve in the container: %w", err)
+	}
+	// copy the python module too
+	globPath := "/usr/lib/*/site-packages/osbuild"
+	matches, err := filepath.Glob(globPath)
+	if err != nil || len(matches) == 0 {
+		return nil, fmt.Errorf("cannot find osbuild python module in %q: %w", globPath, err)
+	}
+	if len(matches) != 1 {
+		return nil, fmt.Errorf("unexpected number of osbuild python module matches: %v", matches)
+	}
+	if err := cnt.CopyInto(matches[0], "/"); err != nil {
+		return nil, fmt.Errorf("cannot prepare depsolve python-modules in the container: %w", err)
+	}
+	return append(cnt.ExecArgv(), "/osbuild-depsolve-dnf"), nil
+}
+
+func (cnt *Container) NewContainerSolver(cacheRoot string, architecture arch.Arch, sourceInfo *source.Info) (*dnfjson.Solver, error) {
+	depsolverCmd, err := cnt.injectDNFJson()
+	if err != nil {
+		return nil, fmt.Errorf("cannot inject depsolve into the container: %w", err)
+	}
+
+	solver := dnfjson.NewSolver(
+		sourceInfo.OSRelease.PlatformID,
+		sourceInfo.OSRelease.VersionID,
+		architecture.String(),
+		fmt.Sprintf("%s-%s", sourceInfo.OSRelease.ID, sourceInfo.OSRelease.VersionID),
+		cacheRoot)
+	solver.SetDNFJSONPath(depsolverCmd[0], depsolverCmd[1:]...)
+	solver.SetRootDir("/")
+	return solver, nil
+}

bib/internal/container/solver_test.go

@@ -0,0 +1,133 @@
+package container_test
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/osbuild/images/pkg/arch"
+	"github.com/osbuild/images/pkg/rpmmd"
+
+	"github.com/osbuild/bootc-image-builder/bib/internal/container"
+	"github.com/osbuild/bootc-image-builder/bib/internal/source"
+)
+
+const (
+	dnfTestingImageRHEL   = "registry.access.redhat.com/ubi9:latest"
+	dnfTestingImageCentos = "quay.io/centos/centos:stream9"
+)
+
+func TestDNFJsonWorks(t *testing.T) {
+	if os.Geteuid() != 0 {
+		t.Skip("skipping test; not running as root")
+	}
+	if _, err := os.Stat("/usr/libexec/osbuild-depsolve-dnf"); err != nil {
+		t.Skip("cannot find /usr/libexec/osbuild-depsolve-dnf")
+	}
+	cacheRoot := t.TempDir()
+
+	cnt, err := container.New(dnfTestingImageCentos)
+	require.NoError(t, err)
+	err = cnt.InitDNF()
+	require.NoError(t, err)
+
+	sourceInfo, err := source.LoadInfo(cnt.Root())
+	require.NoError(t, err)
+	solver, err := cnt.NewContainerSolver(cacheRoot, arch.Current(), sourceInfo)
+	require.NoError(t, err)
+	res, err := solver.Depsolve([]rpmmd.PackageSet{
+		{
+			Include: []string{"coreutils"},
+		},
+	}, 0)
+	require.NoError(t, err)
+	assert.True(t, len(res.Packages) > 0)
+}
+
+func subscribeMachine(t *testing.T) (restore func()) {
+	if _, err := exec.LookPath("subscription-manager"); err != nil {
+		t.Skip("no subscription-manager found")
+		return func() {}
+	}
+
+	matches, err := filepath.Glob("/etc/pki/entitlement/*.pem")
+	if err == nil && len(matches) > 0 {
+		return func() {}
+	}
+
+	rhsmOrg := os.Getenv("RHSM_ORG")
+	rhsmActivationKey := os.Getenv("RHSM_ACTIVATION_KEY")
+	if rhsmOrg == "" || rhsmActivationKey == "" {
+		t.Skip("no RHSM_{ORG,ACTIVATION_KEY} env vars found")
+		return func() {}
+	}
+
+	err = exec.Command("subscription-manager", "register",
+		"--org", rhsmOrg,
+		"--activationkey", rhsmActivationKey).Run()
+	require.NoError(t, err)
+
+	return func() {
+		err := exec.Command("subscription-manager", "unregister").Run()
+		require.NoError(t, err)
+	}
+}
+
+func TestDNFInitGivesAccessToSubscribedContent(t *testing.T) {
+	if os.Geteuid() != 0 {
+		t.Skip("skipping test; not running as root")
+	}
+	if runtime.GOARCH != "amd64" {
+		t.Skip("skipping test; only runs on x86_64")
+	}
+
+	restore := subscribeMachine(t)
+	defer restore()
+
+	cnt, err := container.New(dnfTestingImageRHEL)
+	require.NoError(t, err)
+	err = cnt.InitDNF()
+	require.NoError(t, err)
+
+	content, err := cnt.ReadFile("/etc/yum.repos.d/redhat.repo")
+	require.NoError(t, err)
+	assert.Contains(t, string(content), "rhel-9-for-x86_64-baseos-rpms")
+}
+
+func TestDNFJsonWorkWithSubscribedContent(t *testing.T) {
+	if os.Geteuid() != 0 {
+		t.Skip("skipping test; not running as root")
+	}
+	if runtime.GOARCH != "amd64" {
+		t.Skip("skipping test; only runs on x86_64")
+	}
+	if _, err := os.Stat("/usr/libexec/osbuild-depsolve-dnf"); err != nil {
+		t.Skip("cannot find /usr/libexec/osbuild-depsolve-dnf")
+	}
+	cacheRoot := t.TempDir()
+
+	restore := subscribeMachine(t)
+	defer restore()
+
+	cnt, err := container.New(dnfTestingImageRHEL)
+	require.NoError(t, err)
+	err = cnt.InitDNF()
+	require.NoError(t, err)
+
+	sourceInfo, err := source.LoadInfo(cnt.Root())
+	require.NoError(t, err)
+	solver, err := cnt.NewContainerSolver(cacheRoot, arch.ARCH_X86_64, sourceInfo)
+	require.NoError(t, err)
+	res, err := solver.Depsolve([]rpmmd.PackageSet{
+		{
+			Include: []string{"coreutils"},
+		},
+	}, 0)
+	require.NoError(t, err)
+	assert.True(t, len(res.Packages) > 0)
+}

bib/internal/source/source.go

@@ -5,8 +5,9 @@ import (
 	"os"
 	"path"
 
-	"github.com/osbuild/images/pkg/distro"
 	"github.com/sirupsen/logrus"
+
+	"github.com/osbuild/images/pkg/distro"
 )
 
 type OSRelease struct {

bib/internal/uploader/aws_test.go

@@ -60,6 +60,11 @@ func TestUploadAndRegisterNoProgressBar(t *testing.T) {
 }
 
 func TestUploadAndRegisterProgressBar(t *testing.T) {
+	if os.Getenv("BIB_TESTING_FARM") == "1" {
+		t.Skip("for inexplicable reasons this test fails in testing farm")
+		return
+	}
+
 	fakeStdout := bytes.NewBuffer(nil)
 	restore := uploader.MockOsStdout(fakeStdout)
 	defer restore()

plans/all.fmf → plans/integration.fmf

@@ -11,6 +11,7 @@ prepare:
   how: install
   package:
     - edk2-aarch64
+    - osbuild-depsolve-dnf
     - podman
     - pytest
     - python3-boto3