From 5961d424bede2b807fb41666cec44b849fc76b26 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 17 Oct 2024 15:58:30 -0400
Subject: [PATCH 1/4] teuthology/suite: print base config before config merge

So it's easier to see its value before any configs are generated.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 teuthology/suite/run.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/teuthology/suite/run.py b/teuthology/suite/run.py
index 9b2b6fa70..7f5a1c94b 100644
--- a/teuthology/suite/run.py
+++ b/teuthology/suite/run.py
@@ -622,6 +622,9 @@ def schedule_suite(self):
         log.debug('Suite %s in %s' % (suite_name, suite_path))
         log.debug(f"subset = {self.args.subset}")
         log.debug(f"no_nested_subset = {self.args.no_nested_subset}")
+        if self.args.dry_run:
+            log.debug("Base job config:\n%s" % self.base_config)
+
         configs = build_matrix(suite_path,
                                subset=self.args.subset,
                                no_nested_subset=self.args.no_nested_subset,
@@ -636,9 +639,6 @@ def schedule_suite(self):
             seed=self.args.seed,
             suite_name=suite_name))
 
-        if self.args.dry_run:
-            log.debug("Base job config:\n%s" % self.base_config)
-
         # create, but do not write, the temp file here, so it can be
         # added to the args in collect_jobs, but not filled until
         # any backtracking is done

From 5bac476685eacea5a62f470286b7af6032f2a687 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Thu, 17 Oct 2024 15:59:07 -0400
Subject: [PATCH 2/4] teuthology/suite: set the kernel branch in the kernel
 config

For inspection by scripts.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 teuthology/suite/run.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/teuthology/suite/run.py b/teuthology/suite/run.py
index 7f5a1c94b..cc6859b73 100644
--- a/teuthology/suite/run.py
+++ b/teuthology/suite/run.py
@@ -167,13 +167,16 @@ def choose_kernel(self):
         # Put together a stanza specifying the kernel hash
         if self.args.kernel_branch == 'distro':
             kernel_hash = 'distro'
+            kernel_branch = 'distro'
         # Skip the stanza if '-k none' is given
         elif self.args.kernel_branch is None or \
              self.args.kernel_branch.lower() == 'none':
             kernel_hash = None
+            kernel_branch = None
         else:
+            kernel_branch = self.args.kernel_branch
             kernel_hash = util.get_gitbuilder_hash(
-                'kernel', self.args.kernel_branch, 'default',
+                'kernel', kernel_branch, 'default',
                 self.args.machine_type, self.args.distro,
                 self.args.distro_version,
             )
@@ -189,7 +192,7 @@ def choose_kernel(self):
 
         if kernel_hash:
             log.info("kernel sha1: {hash}".format(hash=kernel_hash))
-            kernel_dict = dict(kernel=dict(kdb=kdb, sha1=kernel_hash))
+            kernel_dict = dict(kernel=dict(branch=kernel_branch, kdb=kdb, sha1=kernel_hash))
             if kernel_hash != 'distro':
                 kernel_dict['kernel']['flavor'] = 'default'
         else:

From d05aa26b35c50545ecebeb0467ec8a68adcc65fa Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Fri, 18 Oct 2024 08:44:13 -0400
Subject: [PATCH 3/4] teuthology/suite: merge base_config with other fragments

Presently the code tries to merge the base_config when the worker starts
running. There's no need to construct it this way and it prevents sharing the
"defaults" with the fragment merging infrastructure. It also prevents
overriding defaults like:

    kernel
        branch: wip-pdonnell-i66704
        client:
            branch: wip-pdonnell-i66704
            flavor: default
            kdb: 1
            sha1: 745cacd8f31e50d7f3b6039bbd8c9a8dfc07bf03
        flavor: default
        kdb: 1
        sha1: 745cacd8f31e50d7f3b6039bbd8c9a8dfc07bf03

A YAML fragment can set kernel.client but it cannot delete the defaults for
kernel.(branch|flavor|kdb|sha1) because there's no way to remove YAML elements
via a deep merge.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 teuthology/suite/merge.py          |  4 ++-
 teuthology/suite/run.py            | 15 +---------
 teuthology/suite/test/test_run_.py | 44 +++++++++++++++++++++++++-----
 3 files changed, 41 insertions(+), 22 deletions(-)

diff --git a/teuthology/suite/merge.py b/teuthology/suite/merge.py
index f13058e47..0e109af02 100644
--- a/teuthology/suite/merge.py
+++ b/teuthology/suite/merge.py
@@ -5,6 +5,7 @@
 from types import MappingProxyType
 import yaml
 
+from teuthology.config import JobConfig
 from teuthology.suite.build_matrix import combine_path
 from teuthology.suite.util import strip_fragment_path
 from teuthology.misc import deep_merge
@@ -115,6 +116,7 @@ def config_merge(configs, suite_name=None, **kwargs):
     the entire job (config) from the list.
     """
     seed = kwargs.setdefault('seed', 1)
+    base_config = kwargs.setdefault('base_config', JobConfig())
     if not isinstance(seed, int):
         log.debug("no valid seed input: using 1")
         seed = 1
@@ -128,7 +130,7 @@ def config_merge(configs, suite_name=None, **kwargs):
         if suite_name is not None:
             desc = combine_path(suite_name, desc)
 
-        yaml_complete_obj = {}
+        yaml_complete_obj = copy.deepcopy(base_config.to_dict())
         deep_merge(yaml_complete_obj, dict(TEUTHOLOGY_TEMPLATE))
         for path in paths:
             if path not in yaml_cache:
diff --git a/teuthology/suite/run.py b/teuthology/suite/run.py
index cc6859b73..ba72a4334 100644
--- a/teuthology/suite/run.py
+++ b/teuthology/suite/run.py
@@ -9,7 +9,6 @@
 
 from humanfriendly import format_timespan
 
-from tempfile import NamedTemporaryFile
 from teuthology import repo_utils
 
 from teuthology.config import config, JobConfig
@@ -639,17 +638,10 @@ def schedule_suite(self):
             filter_out=self.args.filter_out,
             filter_all=self.args.filter_all,
             filter_fragments=self.args.filter_fragments,
+            base_config=self.base_config,
             seed=self.args.seed,
             suite_name=suite_name))
 
-        # create, but do not write, the temp file here, so it can be
-        # added to the args in collect_jobs, but not filled until
-        # any backtracking is done
-        base_yaml_path = NamedTemporaryFile(
-            prefix='schedule_suite_', delete=False
-        ).name
-        self.base_yaml_paths.insert(0, base_yaml_path)
-
         # compute job limit in respect of --sleep-before-teardown
         job_limit = self.args.limit or 0
         sleep_before_teardown = int(self.args.sleep_before_teardown or 0)
@@ -714,9 +706,6 @@ def schedule_suite(self):
                     dry_run=self.args.dry_run,
                 )
 
-        with open(base_yaml_path, 'w+b') as base_yaml:
-            base_yaml.write(str(self.base_config).encode())
-
         if jobs_to_schedule:
             self.write_rerun_memo()
 
@@ -728,8 +717,6 @@ def schedule_suite(self):
 
         self.schedule_jobs(jobs_missing_packages, jobs_to_schedule, name)
 
-        os.remove(base_yaml_path)
-
         count = len(jobs_to_schedule)
         missing_count = len(jobs_missing_packages)
         total_count = count
diff --git a/teuthology/suite/test/test_run_.py b/teuthology/suite/test/test_run_.py
index 0fad56ddf..c97e26e5c 100644
--- a/teuthology/suite/test/test_run_.py
+++ b/teuthology/suite/test/test_run_.py
@@ -1,3 +1,4 @@
+import logging
 import os
 import pytest
 import requests
@@ -14,6 +15,7 @@
 from teuthology.suite import run
 from teuthology.util.time import TIMESTAMP_FMT
 
+log = logging.getLogger(__name__)
 
 class TestRun(object):
     klass = run.Run
@@ -270,14 +272,35 @@ def test_successful_schedule(
             [call('ceph_sha1', 'default', 'ubuntu', '14.04', 'machine_type')],
         )
         y = {
-          'teuthology': {
-            'fragments_dropped': [],
-            'meta': {},
-            'postmerge': []
-          },
           'field1': 'val1',
           'field2': 'val2'
         }
+        teuthology_keys = [
+          'branch',
+          'machine_type',
+          'name',
+          'os_type',
+          'os_version',
+          'overrides',
+          'priority',
+          'repo',
+          'seed',
+          'sha1',
+          'sleep_before_teardown',
+          'suite',
+          'suite_branch',
+          'suite_relpath',
+          'suite_repo',
+          'suite_sha1',
+          'tasks',
+          'teuthology_branch',
+          'teuthology_sha1',
+          'timestamp',
+          'user',
+          'teuthology',
+        ]
+        for t in teuthology_keys:
+            y[t] = ANY
         expected_job = dict(
             yaml=y,
             sha1='ceph_sha1',
@@ -287,16 +310,23 @@ def test_successful_schedule(
                 '--description',
                 os.path.join(self.args.suite, build_matrix_desc),
                 '--',
-                ANY, # base config
                 '-'
             ],
-            stdin=yaml.dump(y),
+            stdin=ANY,
             desc=os.path.join(self.args.suite, build_matrix_desc),
         )
 
         m_schedule_jobs.assert_has_calls(
             [call([], [expected_job], runobj.name)],
         )
+        args = m_schedule_jobs.call_args.args
+        log.debug("args =\n%s", args)
+        jobargs  = args[1][0]
+        stdin_yaml = yaml.safe_load(jobargs['stdin'])
+        for k in y:
+            assert y[k] == stdin_yaml[k]
+        for k in teuthology_keys:
+            assert k in stdin_yaml
         m_write_rerun_memo.assert_called_once_with()
 
     @patch('teuthology.suite.util.find_git_parents')

From 65f6bfacd0412101c447a0b36256ca4d299d82c3 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Sat, 19 Oct 2024 13:39:38 -0400
Subject: [PATCH 4/4] teuthology/tasks/kernel: default branch should be
 "distro"

Generally, we don't want to install any kernel by default. There also is no
"main" branch for the ceph-client.git project.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 teuthology/task/kernel.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/teuthology/task/kernel.py b/teuthology/task/kernel.py
index 59242d31b..22cd1a69c 100644
--- a/teuthology/task/kernel.py
+++ b/teuthology/task/kernel.py
@@ -33,7 +33,7 @@
 
 log = logging.getLogger(__name__)
 
-CONFIG_DEFAULT = {'branch': 'main'}
+CONFIG_DEFAULT = {'branch': 'distro', 'sha1': 'distro'}
 TIMEOUT_DEFAULT = 300
 
 VERSION_KEYS = ['branch', 'tag', 'sha1', 'deb', 'rpm', 'koji', 'koji_task']